[Misc] Reorganize inputs (#35182)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+3
-12
@@ -27,11 +27,9 @@ LLM Class.
|
|||||||
|
|
||||||
- [vllm.LLM][]
|
- [vllm.LLM][]
|
||||||
|
|
||||||
LLM Inputs.
|
Prompt schema for LLM APIs.
|
||||||
|
|
||||||
- [vllm.inputs.PromptType][]
|
- [vllm.inputs.llm][]
|
||||||
- [vllm.inputs.TextPrompt][]
|
|
||||||
- [vllm.inputs.TokensPrompt][]
|
|
||||||
|
|
||||||
## vLLM Engines
|
## vLLM Engines
|
||||||
|
|
||||||
@@ -58,13 +56,7 @@ Looking to add your own multi-modal model? Please follow the instructions listed
|
|||||||
|
|
||||||
- [vllm.multimodal.MULTIMODAL_REGISTRY][]
|
- [vllm.multimodal.MULTIMODAL_REGISTRY][]
|
||||||
|
|
||||||
### Inputs
|
### Internal data structures
|
||||||
|
|
||||||
User-facing inputs.
|
|
||||||
|
|
||||||
- [vllm.multimodal.inputs.MultiModalDataDict][]
|
|
||||||
|
|
||||||
Internal data structures.
|
|
||||||
|
|
||||||
- [vllm.multimodal.inputs.PlaceholderRange][]
|
- [vllm.multimodal.inputs.PlaceholderRange][]
|
||||||
- [vllm.multimodal.inputs.NestedTensors][]
|
- [vllm.multimodal.inputs.NestedTensors][]
|
||||||
@@ -72,7 +64,6 @@ Internal data structures.
|
|||||||
- [vllm.multimodal.inputs.MultiModalFieldConfig][]
|
- [vllm.multimodal.inputs.MultiModalFieldConfig][]
|
||||||
- [vllm.multimodal.inputs.MultiModalKwargsItem][]
|
- [vllm.multimodal.inputs.MultiModalKwargsItem][]
|
||||||
- [vllm.multimodal.inputs.MultiModalKwargsItems][]
|
- [vllm.multimodal.inputs.MultiModalKwargsItems][]
|
||||||
- [vllm.multimodal.inputs.MultiModalInputs][]
|
|
||||||
|
|
||||||
### Data Parsing
|
### Data Parsing
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ Declare supported languages and capabilities:
|
|||||||
from torch import nn
|
from torch import nn
|
||||||
|
|
||||||
from vllm.config import ModelConfig, SpeechToTextConfig
|
from vllm.config import ModelConfig, SpeechToTextConfig
|
||||||
from vllm.inputs.data import PromptType
|
from vllm.inputs import PromptType
|
||||||
from vllm.model_executor.models.interfaces import SupportsTranscription
|
from vllm.model_executor.models.interfaces import SupportsTranscription
|
||||||
|
|
||||||
class YourASRModel(nn.Module, SupportsTranscription):
|
class YourASRModel(nn.Module, SupportsTranscription):
|
||||||
@@ -66,7 +66,7 @@ This is for controlling general behavior of the API when serving your model:
|
|||||||
|
|
||||||
See [Audio preprocessing and chunking](#audio-preprocessing-and-chunking) for what each field controls.
|
See [Audio preprocessing and chunking](#audio-preprocessing-and-chunking) for what each field controls.
|
||||||
|
|
||||||
Implement the prompt construction via [get_generation_prompt][vllm.model_executor.models.interfaces.SupportsTranscription.get_generation_prompt]. The server passes you the resampled waveform and task parameters; you return a valid [PromptType][vllm.inputs.data.PromptType]. There are two common patterns:
|
Implement the prompt construction via [get_generation_prompt][vllm.model_executor.models.interfaces.SupportsTranscription.get_generation_prompt]. The server passes you the resampled waveform and task parameters; you return a valid [PromptType][vllm.inputs.llm.PromptType]. There are two common patterns:
|
||||||
|
|
||||||
#### Multimodal LLM with audio embeddings (e.g., Voxtral, Gemma3n)
|
#### Multimodal LLM with audio embeddings (e.g., Voxtral, Gemma3n)
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ This page teaches you how to pass multi-modal inputs to [multi-modal models](../
|
|||||||
To input multi-modal data, follow this schema in [vllm.inputs.PromptType][]:
|
To input multi-modal data, follow this schema in [vllm.inputs.PromptType][]:
|
||||||
|
|
||||||
- `prompt`: The prompt should follow the format that is documented on HuggingFace.
|
- `prompt`: The prompt should follow the format that is documented on HuggingFace.
|
||||||
- `multi_modal_data`: This is a dictionary that follows the schema defined in [vllm.multimodal.inputs.MultiModalDataDict][].
|
- `multi_modal_data`: This is a dictionary that follows the schema defined in [vllm.inputs.MultiModalDataDict][].
|
||||||
|
|
||||||
### Image Inputs
|
### Image Inputs
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm import LLM
|
from vllm import LLM
|
||||||
from vllm.inputs.data import TextPrompt
|
from vllm.inputs import TextPrompt
|
||||||
from vllm.multimodal.utils import fetch_image
|
from vllm.multimodal.utils import fetch_image
|
||||||
|
|
||||||
# Initialize model
|
# Initialize model
|
||||||
|
|||||||
@@ -105,7 +105,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def _fake_preprocess_chat(*args, **kwargs):
|
async def _fake_preprocess_chat(*args, **kwargs):
|
||||||
# return conversation, engine_prompts
|
# return conversation, engine_inputs
|
||||||
return (
|
return (
|
||||||
[{"role": "user", "content": "Test"}],
|
[{"role": "user", "content": "Test"}],
|
||||||
[{"prompt_token_ids": [1, 2, 3]}],
|
[{"prompt_token_ids": [1, 2, 3]}],
|
||||||
|
|||||||
@@ -958,14 +958,14 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):
|
|||||||
serving_chat = _build_serving_chat(mock_engine)
|
serving_chat = _build_serving_chat(mock_engine)
|
||||||
|
|
||||||
orig_render_chat_request = serving_chat.render_chat_request
|
orig_render_chat_request = serving_chat.render_chat_request
|
||||||
captured_prompts = []
|
captured_inputs = []
|
||||||
|
|
||||||
async def render_chat_request(request):
|
async def render_chat_request(request):
|
||||||
result = await orig_render_chat_request(request)
|
result = await orig_render_chat_request(request)
|
||||||
|
|
||||||
assert isinstance(result, tuple)
|
assert isinstance(result, tuple)
|
||||||
conversation, engine_prompts = result
|
conversation, engine_inputs = result
|
||||||
captured_prompts.extend(engine_prompts)
|
captured_inputs.extend(engine_inputs)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@@ -981,18 +981,18 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):
|
|||||||
with suppress(Exception):
|
with suppress(Exception):
|
||||||
await serving_chat.create_chat_completion(req)
|
await serving_chat.create_chat_completion(req)
|
||||||
|
|
||||||
assert len(captured_prompts) == 1
|
assert len(captured_inputs) == 1
|
||||||
assert "cache_salt" not in captured_prompts[0]
|
assert "cache_salt" not in captured_inputs[0]
|
||||||
|
|
||||||
captured_prompts.clear()
|
captured_inputs.clear()
|
||||||
|
|
||||||
# Test with certain cache_salt
|
# Test with certain cache_salt
|
||||||
req.cache_salt = "test_salt"
|
req.cache_salt = "test_salt"
|
||||||
with suppress(Exception):
|
with suppress(Exception):
|
||||||
await serving_chat.create_chat_completion(req)
|
await serving_chat.create_chat_completion(req)
|
||||||
|
|
||||||
assert len(captured_prompts) == 1
|
assert len(captured_inputs) == 1
|
||||||
assert captured_prompts[0]["cache_salt"] == "test_salt"
|
assert captured_inputs[0]["cache_salt"] == "test_salt"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ from vllm.entrypoints.openai.responses.serving import (
|
|||||||
from vllm.entrypoints.openai.responses.streaming_events import (
|
from vllm.entrypoints.openai.responses.streaming_events import (
|
||||||
StreamingState,
|
StreamingState,
|
||||||
)
|
)
|
||||||
from vllm.inputs.data import TokensPrompt
|
from vllm.inputs import tokens_input
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
|
||||||
@@ -258,20 +258,20 @@ class TestValidateGeneratorInput:
|
|||||||
"""Test _validate_generator_input with valid prompt length"""
|
"""Test _validate_generator_input with valid prompt length"""
|
||||||
# Create an engine prompt with valid length (less than max_model_len)
|
# Create an engine prompt with valid length (less than max_model_len)
|
||||||
valid_prompt_token_ids = list(range(5)) # 5 tokens < 100 max_model_len
|
valid_prompt_token_ids = list(range(5)) # 5 tokens < 100 max_model_len
|
||||||
engine_prompt = TokensPrompt(prompt_token_ids=valid_prompt_token_ids)
|
engine_input = tokens_input(valid_prompt_token_ids)
|
||||||
|
|
||||||
# Call the method
|
# Call the method
|
||||||
result = serving_responses_instance._validate_generator_input(engine_prompt)
|
result = serving_responses_instance._validate_generator_input(engine_input)
|
||||||
|
|
||||||
# Should return None for valid input
|
# Should return None for valid input
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
# create an invalid engine prompt
|
# create an invalid engine prompt
|
||||||
invalid_prompt_token_ids = list(range(200)) # 100 tokens >= 100 max_model_len
|
invalid_prompt_token_ids = list(range(200)) # 100 tokens >= 100 max_model_len
|
||||||
engine_prompt = TokensPrompt(prompt_token_ids=invalid_prompt_token_ids)
|
engine_input = tokens_input(invalid_prompt_token_ids)
|
||||||
|
|
||||||
# Call the method
|
# Call the method
|
||||||
result = serving_responses_instance._validate_generator_input(engine_prompt)
|
result = serving_responses_instance._validate_generator_input(engine_input)
|
||||||
|
|
||||||
# Should return an ErrorResponse
|
# Should return an ErrorResponse
|
||||||
assert result is not None
|
assert result is not None
|
||||||
|
|||||||
@@ -73,20 +73,6 @@ async def test_chat_render_multi_turn(client):
|
|||||||
assert len(data["token_ids"]) > 0
|
assert len(data["token_ids"]) > 0
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_chat_render_invalid_model(client):
|
|
||||||
response = await client.post(
|
|
||||||
"/v1/chat/completions/render",
|
|
||||||
json={
|
|
||||||
"model": "nonexistent-model",
|
|
||||||
"messages": [{"role": "user", "content": "Hello"}],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 404
|
|
||||||
assert "error" in response.json()
|
|
||||||
|
|
||||||
|
|
||||||
# -- Completion Render --
|
# -- Completion Render --
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
parse_chat_messages,
|
parse_chat_messages,
|
||||||
parse_chat_messages_async,
|
parse_chat_messages_async,
|
||||||
)
|
)
|
||||||
from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict
|
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||||
from vllm.multimodal.utils import (
|
from vllm.multimodal.utils import (
|
||||||
encode_audio_url,
|
encode_audio_url,
|
||||||
encode_image_url,
|
encode_image_url,
|
||||||
|
|||||||
@@ -13,8 +13,8 @@ from mistral_common.tokens.tokenizers.multimodal import image_from_chunk
|
|||||||
from transformers import AutoProcessor
|
from transformers import AutoProcessor
|
||||||
|
|
||||||
from vllm import SamplingParams, TextPrompt, TokensPrompt
|
from vllm import SamplingParams, TextPrompt, TokensPrompt
|
||||||
|
from vllm.inputs import MultiModalDataBuiltins
|
||||||
from vllm.logprobs import Logprob, SampleLogprobs
|
from vllm.logprobs import Logprob, SampleLogprobs
|
||||||
from vllm.multimodal import MultiModalDataBuiltins
|
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
from ....utils import VLLM_PATH, large_gpu_test
|
from ....utils import VLLM_PATH, large_gpu_test
|
||||||
|
|||||||
@@ -15,13 +15,11 @@ from vllm.config.multimodal import (
|
|||||||
ImageDummyOptions,
|
ImageDummyOptions,
|
||||||
VideoDummyOptions,
|
VideoDummyOptions,
|
||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
|
from vllm.inputs import MultiModalDataDict, MultiModalInput
|
||||||
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||||
from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal
|
from vllm.multimodal.inputs import batched_tensors_equal
|
||||||
from vllm.multimodal.processing import (
|
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
|
||||||
BaseMultiModalProcessor,
|
|
||||||
InputProcessingContext,
|
|
||||||
)
|
|
||||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||||
from vllm.utils.mistral import is_mistral_tokenizer
|
from vllm.utils.mistral import is_mistral_tokenizer
|
||||||
|
|
||||||
@@ -420,8 +418,8 @@ def test_processing_correctness(
|
|||||||
|
|
||||||
|
|
||||||
def _assert_inputs_equal(
|
def _assert_inputs_equal(
|
||||||
a: MultiModalInputs,
|
a: MultiModalInput,
|
||||||
b: MultiModalInputs,
|
b: MultiModalInput,
|
||||||
*,
|
*,
|
||||||
ignore_mm_keys: set[str] | None = None,
|
ignore_mm_keys: set[str] | None = None,
|
||||||
msg: str = "",
|
msg: str = "",
|
||||||
|
|||||||
+2
-4
@@ -6,11 +6,9 @@ from collections.abc import Sequence
|
|||||||
from vllm.config import ModelConfig, PoolerConfig, VllmConfig
|
from vllm.config import ModelConfig, PoolerConfig, VllmConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
||||||
from vllm.entrypoints.pooling.base.protocol import EmbedRequestMixin
|
from vllm.entrypoints.pooling.base.protocol import EmbedRequestMixin
|
||||||
from vllm.inputs.data import PromptType
|
from vllm.inputs import PromptType
|
||||||
from vllm.outputs import PoolingRequestOutput
|
from vllm.outputs import PoolingRequestOutput
|
||||||
from vllm.plugins.io_processors.interface import (
|
from vllm.plugins.io_processors.interface import IOProcessor
|
||||||
IOProcessor,
|
|
||||||
)
|
|
||||||
from vllm.pooling_params import PoolingParams
|
from vllm.pooling_params import PoolingParams
|
||||||
from vllm.renderers import BaseRenderer
|
from vllm.renderers import BaseRenderer
|
||||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from einops import rearrange
|
|||||||
from terratorch.datamodules import Sen1Floods11NonGeoDataModule
|
from terratorch.datamodules import Sen1Floods11NonGeoDataModule
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.inputs.data import PromptType
|
from vllm.inputs import PromptType
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.outputs import PoolingRequestOutput
|
from vllm.outputs import PoolingRequestOutput
|
||||||
from vllm.plugins.io_processors.interface import IOProcessor
|
from vllm.plugins.io_processors.interface import IOProcessor
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from unittest.mock import MagicMock, patch
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.inputs.data import PromptType
|
from vllm.inputs import PromptType
|
||||||
from vllm.outputs import PoolingRequestOutput
|
from vllm.outputs import PoolingRequestOutput
|
||||||
from vllm.plugins.io_processors import get_io_processor
|
from vllm.plugins.io_processors import get_io_processor
|
||||||
from vllm.plugins.io_processors.interface import IOProcessor
|
from vllm.plugins.io_processors.interface import IOProcessor
|
||||||
|
|||||||
@@ -15,13 +15,13 @@ def test_text_input():
|
|||||||
assert prompt_to_seq(["foo", "bar"]) == ["foo", "bar"]
|
assert prompt_to_seq(["foo", "bar"]) == ["foo", "bar"]
|
||||||
|
|
||||||
|
|
||||||
def test_token_input():
|
def test_tokens_input():
|
||||||
assert prompt_to_seq([1, 2]) == [[1, 2]]
|
assert prompt_to_seq([1, 2]) == [[1, 2]]
|
||||||
assert prompt_to_seq([[1, 2]]) == [[1, 2]]
|
assert prompt_to_seq([[1, 2]]) == [[1, 2]]
|
||||||
assert prompt_to_seq([[1, 2], [3, 4]]) == [[1, 2], [3, 4]]
|
assert prompt_to_seq([[1, 2], [3, 4]]) == [[1, 2], [3, 4]]
|
||||||
|
|
||||||
|
|
||||||
def test_text_token_input():
|
def test_text_tokens_input():
|
||||||
assert prompt_to_seq([[1, 2], "foo"]) == [[1, 2], "foo"]
|
assert prompt_to_seq([[1, 2], "foo"]) == [[1, 2], "foo"]
|
||||||
assert prompt_to_seq(["foo", [1, 2]]) == ["foo", [1, 2]]
|
assert prompt_to_seq(["foo", [1, 2]]) == ["foo", [1, 2]]
|
||||||
|
|
||||||
|
|||||||
@@ -129,7 +129,7 @@ class TestValidatePrompt:
|
|||||||
|
|
||||||
|
|
||||||
class TestRenderPrompt:
|
class TestRenderPrompt:
|
||||||
def test_token_input(self):
|
def test_tokens_input(self):
|
||||||
renderer = _build_renderer(MockModelConfig())
|
renderer = _build_renderer(MockModelConfig())
|
||||||
|
|
||||||
tokens = [101, 7592, 2088]
|
tokens = [101, 7592, 2088]
|
||||||
@@ -339,7 +339,7 @@ class TestRenderPrompt:
|
|||||||
TokenizeParams(max_total_tokens=100),
|
TokenizeParams(max_total_tokens=100),
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_token_input_with_needs_detokenization(self):
|
def test_tokens_input_with_needs_detokenization(self):
|
||||||
renderer = _build_renderer(MockModelConfig())
|
renderer = _build_renderer(MockModelConfig())
|
||||||
|
|
||||||
tokens = [1, 2, 3, 4]
|
tokens = [1, 2, 3, 4]
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import pytest
|
|||||||
from tests.v1.shutdown.utils import SHUTDOWN_TEST_TIMEOUT_SEC
|
from tests.v1.shutdown.utils import SHUTDOWN_TEST_TIMEOUT_SEC
|
||||||
from vllm import SamplingParams
|
from vllm import SamplingParams
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||||
from vllm.inputs.data import TokensPrompt
|
from vllm.inputs import TokensPrompt
|
||||||
from vllm.sampling_params import RequestOutputKind
|
from vllm.sampling_params import RequestOutputKind
|
||||||
from vllm.v1.engine.async_llm import AsyncLLM
|
from vllm.v1.engine.async_llm import AsyncLLM
|
||||||
from vllm.v1.engine.exceptions import EngineGenerateError
|
from vllm.v1.engine.exceptions import EngineGenerateError
|
||||||
|
|||||||
+18
-13
@@ -3,11 +3,16 @@
|
|||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from vllm.inputs import EncoderDecoderInputs, TokenInputs, token_inputs
|
from vllm.inputs import (
|
||||||
from vllm.inputs.data import DecoderInputs
|
DecoderOnlyEngineInput,
|
||||||
|
EncoderDecoderInput,
|
||||||
|
MultiModalInput,
|
||||||
|
TokensInput,
|
||||||
|
mm_input,
|
||||||
|
tokens_input,
|
||||||
|
)
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
from vllm.multimodal.inputs import MultiModalInputs, mm_inputs
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -18,7 +23,7 @@ class BeamSearchSequence:
|
|||||||
about to be returned to the user.
|
about to be returned to the user.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
orig_prompt: TokenInputs | MultiModalInputs | EncoderDecoderInputs
|
orig_prompt: TokensInput | MultiModalInput | EncoderDecoderInput
|
||||||
|
|
||||||
# NOTE: Tokens represents decoder tokens in the encoder / decoder case
|
# NOTE: Tokens represents decoder tokens in the encoder / decoder case
|
||||||
tokens: list[int]
|
tokens: list[int]
|
||||||
@@ -40,13 +45,13 @@ class BeamSearchSequence:
|
|||||||
cache_salt = prompt.get("cache_salt")
|
cache_salt = prompt.get("cache_salt")
|
||||||
|
|
||||||
if prompt["type"] == "token":
|
if prompt["type"] == "token":
|
||||||
return token_inputs(
|
return tokens_input(
|
||||||
self.tokens,
|
self.tokens,
|
||||||
prompt=prompt_text,
|
prompt=prompt_text,
|
||||||
cache_salt=cache_salt,
|
cache_salt=cache_salt,
|
||||||
)
|
)
|
||||||
|
|
||||||
return mm_inputs(
|
return mm_input(
|
||||||
prompt_token_ids=self.tokens,
|
prompt_token_ids=self.tokens,
|
||||||
mm_kwargs=prompt["mm_kwargs"],
|
mm_kwargs=prompt["mm_kwargs"],
|
||||||
mm_hashes=prompt["mm_hashes"],
|
mm_hashes=prompt["mm_hashes"],
|
||||||
@@ -56,8 +61,8 @@ class BeamSearchSequence:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def _build_encoder_decoder_inputs(
|
def _build_encoder_decoder_inputs(
|
||||||
self, prompt: EncoderDecoderInputs
|
self, prompt: EncoderDecoderInput
|
||||||
) -> EncoderDecoderInputs:
|
) -> EncoderDecoderInput:
|
||||||
"""Rebuild the encoder-decoder inputs with the current beam search
|
"""Rebuild the encoder-decoder inputs with the current beam search
|
||||||
sequence's tokens.
|
sequence's tokens.
|
||||||
|
|
||||||
@@ -70,9 +75,9 @@ class BeamSearchSequence:
|
|||||||
|
|
||||||
# Rebuild decoder prompt with updated tokens,
|
# Rebuild decoder prompt with updated tokens,
|
||||||
# but keep everything else the same.
|
# but keep everything else the same.
|
||||||
new_dec_prompt: DecoderInputs
|
new_dec_prompt: DecoderOnlyEngineInput
|
||||||
if dec_prompt["type"] == "multimodal":
|
if dec_prompt["type"] == "multimodal":
|
||||||
new_dec_prompt = mm_inputs(
|
new_dec_prompt = mm_input(
|
||||||
self.tokens,
|
self.tokens,
|
||||||
mm_kwargs=dec_prompt["mm_kwargs"],
|
mm_kwargs=dec_prompt["mm_kwargs"],
|
||||||
mm_hashes=dec_prompt["mm_hashes"],
|
mm_hashes=dec_prompt["mm_hashes"],
|
||||||
@@ -81,13 +86,13 @@ class BeamSearchSequence:
|
|||||||
cache_salt=dec_prompt.get("cache_salt"),
|
cache_salt=dec_prompt.get("cache_salt"),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
new_dec_prompt = token_inputs(
|
new_dec_prompt = tokens_input(
|
||||||
self.tokens,
|
self.tokens,
|
||||||
prompt=dec_prompt.get("prompt"),
|
prompt=dec_prompt.get("prompt"),
|
||||||
cache_salt=dec_prompt.get("cache_salt"),
|
cache_salt=dec_prompt.get("cache_salt"),
|
||||||
)
|
)
|
||||||
|
|
||||||
return EncoderDecoderInputs(
|
return EncoderDecoderInput(
|
||||||
type="enc_dec",
|
type="enc_dec",
|
||||||
encoder_prompt=prompt["encoder_prompt"],
|
encoder_prompt=prompt["encoder_prompt"],
|
||||||
decoder_prompt=new_dec_prompt,
|
decoder_prompt=new_dec_prompt,
|
||||||
@@ -107,7 +112,7 @@ class BeamSearchOutput:
|
|||||||
class BeamSearchInstance:
|
class BeamSearchInstance:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
prompt: TokenInputs | MultiModalInputs | EncoderDecoderInputs,
|
prompt: TokensInput | MultiModalInput | EncoderDecoderInput,
|
||||||
lora_request: LoRARequest | None = None,
|
lora_request: LoRARequest | None = None,
|
||||||
logprobs: list[dict[int, Logprob]] | None = None,
|
logprobs: list[dict[int, Logprob]] | None = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
|||||||
@@ -35,9 +35,9 @@ from huggingface_hub import snapshot_download
|
|||||||
from PIL import Image
|
from PIL import Image
|
||||||
from typing_extensions import deprecated
|
from typing_extensions import deprecated
|
||||||
|
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
from vllm.lora.utils import get_adapter_absolute_path
|
from vllm.lora.utils import get_adapter_absolute_path
|
||||||
from vllm.multimodal import MultiModalDataDict
|
|
||||||
from vllm.multimodal.audio import get_audio_duration
|
from vllm.multimodal.audio import get_audio_duration
|
||||||
from vllm.multimodal.image import convert_image_mode
|
from vllm.multimodal.image import convert_image_mode
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from vllm.distributed.weight_transfer.base import (
|
|||||||
WeightTransferInitRequest,
|
WeightTransferInitRequest,
|
||||||
WeightTransferUpdateRequest,
|
WeightTransferUpdateRequest,
|
||||||
)
|
)
|
||||||
from vllm.inputs.data import ProcessorInputs, PromptType
|
from vllm.inputs import EngineInput, PromptType
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
from vllm.outputs import PoolingRequestOutput, RequestOutput
|
from vllm.outputs import PoolingRequestOutput, RequestOutput
|
||||||
from vllm.plugins.io_processors import IOProcessor
|
from vllm.plugins.io_processors import IOProcessor
|
||||||
@@ -34,7 +34,7 @@ class StreamingInput:
|
|||||||
where inputs are provided via an async generator.
|
where inputs are provided via an async generator.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
prompt: ProcessorInputs
|
prompt: EngineInput
|
||||||
sampling_params: SamplingParams | None = None
|
sampling_params: SamplingParams | None = None
|
||||||
|
|
||||||
|
|
||||||
@@ -68,7 +68,7 @@ class EngineClient(ABC):
|
|||||||
self,
|
self,
|
||||||
prompt: EngineCoreRequest
|
prompt: EngineCoreRequest
|
||||||
| PromptType
|
| PromptType
|
||||||
| ProcessorInputs
|
| EngineInput
|
||||||
| AsyncGenerator[StreamingInput, None],
|
| AsyncGenerator[StreamingInput, None],
|
||||||
sampling_params: SamplingParams,
|
sampling_params: SamplingParams,
|
||||||
request_id: str,
|
request_id: str,
|
||||||
@@ -87,7 +87,7 @@ class EngineClient(ABC):
|
|||||||
@abstractmethod
|
@abstractmethod
|
||||||
def encode(
|
def encode(
|
||||||
self,
|
self,
|
||||||
prompt: PromptType | ProcessorInputs,
|
prompt: PromptType | EngineInput,
|
||||||
pooling_params: PoolingParams,
|
pooling_params: PoolingParams,
|
||||||
request_id: str,
|
request_id: str,
|
||||||
lora_request: LoRARequest | None = None,
|
lora_request: LoRARequest | None = None,
|
||||||
|
|||||||
@@ -797,12 +797,12 @@ class AnthropicServingMessages(OpenAIServingChat):
|
|||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
_, engine_prompts = result
|
_, engine_inputs = result
|
||||||
|
|
||||||
input_tokens = sum( # type: ignore
|
input_tokens = sum( # type: ignore
|
||||||
len(prompt["prompt_token_ids"]) # type: ignore[typeddict-item, misc]
|
len(engine_input["prompt_token_ids"]) # type: ignore[typeddict-item, misc]
|
||||||
for prompt in engine_prompts
|
for engine_input in engine_inputs
|
||||||
if "prompt_token_ids" in prompt
|
if "prompt_token_ids" in engine_input
|
||||||
)
|
)
|
||||||
|
|
||||||
response = AnthropicCountTokensResponse(
|
response = AnthropicCountTokensResponse(
|
||||||
|
|||||||
@@ -40,9 +40,10 @@ from typing_extensions import Required, TypedDict
|
|||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
|
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.models import SupportsMultiModal
|
from vllm.model_executor.models import SupportsMultiModal
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalBatchedField,
|
MultiModalBatchedField,
|
||||||
MultiModalFlatField,
|
MultiModalFlatField,
|
||||||
|
|||||||
+22
-22
@@ -57,9 +57,9 @@ from vllm.entrypoints.pooling.score.utils import (
|
|||||||
validate_score_input,
|
validate_score_input,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.utils import log_non_default_args
|
from vllm.entrypoints.utils import log_non_default_args
|
||||||
from vllm.inputs.data import (
|
from vllm.inputs import (
|
||||||
DataPrompt,
|
DataPrompt,
|
||||||
ProcessorInputs,
|
EngineInput,
|
||||||
PromptType,
|
PromptType,
|
||||||
SingletonPrompt,
|
SingletonPrompt,
|
||||||
TextPrompt,
|
TextPrompt,
|
||||||
@@ -589,7 +589,7 @@ class LLM:
|
|||||||
|
|
||||||
def _resolve_mm_lora(
|
def _resolve_mm_lora(
|
||||||
self,
|
self,
|
||||||
prompt: ProcessorInputs,
|
prompt: EngineInput,
|
||||||
lora_request: LoRARequest | None,
|
lora_request: LoRARequest | None,
|
||||||
) -> LoRARequest | None:
|
) -> LoRARequest | None:
|
||||||
if prompt["type"] != "multimodal":
|
if prompt["type"] != "multimodal":
|
||||||
@@ -716,8 +716,8 @@ class LLM:
|
|||||||
eos_token_id = tokenizer.eos_token_id
|
eos_token_id = tokenizer.eos_token_id
|
||||||
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
|
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
|
||||||
|
|
||||||
engine_prompts = self._preprocess_cmpl(prompts)
|
engine_inputs = self._preprocess_cmpl(prompts)
|
||||||
lora_requests = self._lora_request_to_seq(lora_request, len(engine_prompts))
|
lora_requests = self._lora_request_to_seq(lora_request, len(engine_inputs))
|
||||||
|
|
||||||
if use_tqdm and concurrency_limit is not None:
|
if use_tqdm and concurrency_limit is not None:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
@@ -727,7 +727,7 @@ class LLM:
|
|||||||
use_tqdm = False
|
use_tqdm = False
|
||||||
|
|
||||||
if concurrency_limit is None:
|
if concurrency_limit is None:
|
||||||
concurrency_limit = len(engine_prompts)
|
concurrency_limit = len(engine_inputs)
|
||||||
|
|
||||||
# generate 2 * beam_width candidates at each step
|
# generate 2 * beam_width candidates at each step
|
||||||
# following the huggingface transformers implementation
|
# following the huggingface transformers implementation
|
||||||
@@ -740,7 +740,7 @@ class LLM:
|
|||||||
)
|
)
|
||||||
instances: list[BeamSearchInstance] = []
|
instances: list[BeamSearchInstance] = []
|
||||||
|
|
||||||
for lora_req, prompt in zip(lora_requests, engine_prompts):
|
for lora_req, prompt in zip(lora_requests, engine_inputs):
|
||||||
if prompt["type"] == "embeds":
|
if prompt["type"] == "embeds":
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Embedding prompt not supported for beam search"
|
"Embedding prompt not supported for beam search"
|
||||||
@@ -845,7 +845,7 @@ class LLM:
|
|||||||
self,
|
self,
|
||||||
prompts: Sequence[PromptType],
|
prompts: Sequence[PromptType],
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> Sequence[ProcessorInputs]:
|
) -> Sequence[EngineInput]:
|
||||||
"""
|
"""
|
||||||
Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
|
Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
|
||||||
a format that can be passed to `_add_request`.
|
a format that can be passed to `_add_request`.
|
||||||
@@ -853,7 +853,7 @@ class LLM:
|
|||||||
Refer to [LLM.generate][] for a complete description of the arguments.
|
Refer to [LLM.generate][] for a complete description of the arguments.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A list of `ProcessorInputs` objects ready to be passed into LLMEngine.
|
A list of `EngineInput` objects ready to be passed into LLMEngine.
|
||||||
"""
|
"""
|
||||||
renderer = self.renderer
|
renderer = self.renderer
|
||||||
model_config = self.model_config
|
model_config = self.model_config
|
||||||
@@ -871,9 +871,9 @@ class LLM:
|
|||||||
self,
|
self,
|
||||||
prompt: PromptType,
|
prompt: PromptType,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> ProcessorInputs:
|
) -> EngineInput:
|
||||||
(engine_prompt,) = self._preprocess_cmpl([prompt], tokenization_kwargs)
|
(engine_input,) = self._preprocess_cmpl([prompt], tokenization_kwargs)
|
||||||
return engine_prompt
|
return engine_input
|
||||||
|
|
||||||
def _preprocess_chat(
|
def _preprocess_chat(
|
||||||
self,
|
self,
|
||||||
@@ -886,7 +886,7 @@ class LLM:
|
|||||||
tools: list[dict[str, Any]] | None = None,
|
tools: list[dict[str, Any]] | None = None,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||||
) -> Sequence[ProcessorInputs]:
|
) -> Sequence[EngineInput]:
|
||||||
"""
|
"""
|
||||||
Convert a list of conversations into prompts so that they can then
|
Convert a list of conversations into prompts so that they can then
|
||||||
be used as input for other LLM APIs.
|
be used as input for other LLM APIs.
|
||||||
@@ -894,7 +894,7 @@ class LLM:
|
|||||||
Refer to [LLM.chat][] for a complete description of the arguments.
|
Refer to [LLM.chat][] for a complete description of the arguments.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A list of `ProcessorInputs` objects ready to be passed into LLMEngine.
|
A list of `EngineInput` objects ready to be passed into LLMEngine.
|
||||||
"""
|
"""
|
||||||
renderer = self.renderer
|
renderer = self.renderer
|
||||||
|
|
||||||
@@ -915,14 +915,14 @@ class LLM:
|
|||||||
**(tokenization_kwargs or {})
|
**(tokenization_kwargs or {})
|
||||||
)
|
)
|
||||||
|
|
||||||
_, engine_prompts = renderer.render_chat(
|
_, engine_inputs = renderer.render_chat(
|
||||||
conversations,
|
conversations,
|
||||||
chat_params,
|
chat_params,
|
||||||
tok_params,
|
tok_params,
|
||||||
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
|
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
|
||||||
)
|
)
|
||||||
|
|
||||||
return engine_prompts
|
return engine_inputs
|
||||||
|
|
||||||
def _preprocess_chat_one(
|
def _preprocess_chat_one(
|
||||||
self,
|
self,
|
||||||
@@ -935,8 +935,8 @@ class LLM:
|
|||||||
tools: list[dict[str, Any]] | None = None,
|
tools: list[dict[str, Any]] | None = None,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||||
) -> ProcessorInputs:
|
) -> EngineInput:
|
||||||
(engine_prompt,) = self._preprocess_chat(
|
(engine_input,) = self._preprocess_chat(
|
||||||
[conversation],
|
[conversation],
|
||||||
chat_template=chat_template,
|
chat_template=chat_template,
|
||||||
chat_template_content_format=chat_template_content_format,
|
chat_template_content_format=chat_template_content_format,
|
||||||
@@ -948,7 +948,7 @@ class LLM:
|
|||||||
mm_processor_kwargs=mm_processor_kwargs,
|
mm_processor_kwargs=mm_processor_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
return engine_prompt
|
return engine_input
|
||||||
|
|
||||||
def chat(
|
def chat(
|
||||||
self,
|
self,
|
||||||
@@ -1909,7 +1909,7 @@ class LLM:
|
|||||||
|
|
||||||
def _render_and_run_requests(
|
def _render_and_run_requests(
|
||||||
self,
|
self,
|
||||||
prompts: Iterable[ProcessorInputs],
|
prompts: Iterable[EngineInput],
|
||||||
params: Sequence[SamplingParams | PoolingParams],
|
params: Sequence[SamplingParams | PoolingParams],
|
||||||
output_type: type[_O],
|
output_type: type[_O],
|
||||||
*,
|
*,
|
||||||
@@ -1938,7 +1938,7 @@ class LLM:
|
|||||||
|
|
||||||
def _render_and_add_requests(
|
def _render_and_add_requests(
|
||||||
self,
|
self,
|
||||||
prompts: Iterable[ProcessorInputs],
|
prompts: Iterable[EngineInput],
|
||||||
params: Sequence[SamplingParams | PoolingParams],
|
params: Sequence[SamplingParams | PoolingParams],
|
||||||
*,
|
*,
|
||||||
lora_requests: Sequence[LoRARequest | None] | None = None,
|
lora_requests: Sequence[LoRARequest | None] | None = None,
|
||||||
@@ -1967,7 +1967,7 @@ class LLM:
|
|||||||
|
|
||||||
def _add_request(
|
def _add_request(
|
||||||
self,
|
self,
|
||||||
prompt: ProcessorInputs,
|
prompt: EngineInput,
|
||||||
params: SamplingParams | PoolingParams,
|
params: SamplingParams | PoolingParams,
|
||||||
lora_request: LoRARequest | None = None,
|
lora_request: LoRARequest | None = None,
|
||||||
priority: int = 0,
|
priority: int = 0,
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
|
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
|
||||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||||
from vllm.inputs.data import ProcessorInputs
|
from vllm.inputs import EngineInput
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
@@ -177,7 +177,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
async def render_chat_request(
|
async def render_chat_request(
|
||||||
self,
|
self,
|
||||||
request: ChatCompletionRequest,
|
request: ChatCompletionRequest,
|
||||||
) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
|
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
|
||||||
"""
|
"""
|
||||||
Validate the model and preprocess a chat completion request.
|
Validate the model and preprocess a chat completion request.
|
||||||
|
|
||||||
@@ -185,7 +185,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
engine-aware checks (LoRA model validation, engine health).
|
engine-aware checks (LoRA model validation, engine health).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A tuple of (conversation, engine_prompts) on success,
|
A tuple of (conversation, engine_inputs) on success,
|
||||||
or an ErrorResponse on failure.
|
or an ErrorResponse on failure.
|
||||||
"""
|
"""
|
||||||
error_check_ret = await self._check_model(request)
|
error_check_ret = await self._check_model(request)
|
||||||
@@ -231,7 +231,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
conversation, engine_prompts = result
|
conversation, engine_inputs = result
|
||||||
|
|
||||||
request_id = (
|
request_id = (
|
||||||
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||||
@@ -251,13 +251,13 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
# Schedule the request and get the result generator.
|
# Schedule the request and get the result generator.
|
||||||
max_model_len = self.model_config.max_model_len
|
max_model_len = self.model_config.max_model_len
|
||||||
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
||||||
for i, engine_prompt in enumerate(engine_prompts):
|
for i, engine_input in enumerate(engine_inputs):
|
||||||
prompt_token_ids = self._extract_prompt_components(engine_prompt).token_ids
|
prompt_token_ids = self._extract_prompt_components(engine_input).token_ids
|
||||||
|
|
||||||
# If we are creating sub requests for multiple prompts, ensure that they
|
# If we are creating sub requests for multiple prompts, ensure that they
|
||||||
# have unique request ids.
|
# have unique request ids.
|
||||||
sub_request_id = (
|
sub_request_id = (
|
||||||
request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
|
request_id if len(engine_inputs) == 1 else f"{request_id}_{i}"
|
||||||
)
|
)
|
||||||
|
|
||||||
max_tokens = get_max_tokens(
|
max_tokens = get_max_tokens(
|
||||||
@@ -265,7 +265,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
request.max_completion_tokens
|
request.max_completion_tokens
|
||||||
if request.max_completion_tokens is not None
|
if request.max_completion_tokens is not None
|
||||||
else request.max_tokens,
|
else request.max_tokens,
|
||||||
self._extract_prompt_len(engine_prompt),
|
self._extract_prompt_len(engine_input),
|
||||||
self.default_sampling_params,
|
self.default_sampling_params,
|
||||||
self.override_max_tokens,
|
self.override_max_tokens,
|
||||||
)
|
)
|
||||||
@@ -283,7 +283,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
sub_request_id,
|
sub_request_id,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=sampling_params,
|
params=sampling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
@@ -296,7 +296,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
|
|
||||||
if isinstance(sampling_params, BeamSearchParams):
|
if isinstance(sampling_params, BeamSearchParams):
|
||||||
generator = self.beam_search(
|
generator = self.beam_search(
|
||||||
prompt=engine_prompt,
|
prompt=engine_input,
|
||||||
request_id=sub_request_id,
|
request_id=sub_request_id,
|
||||||
params=sampling_params,
|
params=sampling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -313,7 +313,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
reasoning_ended = None
|
reasoning_ended = None
|
||||||
|
|
||||||
generator = self.engine_client.generate(
|
generator = self.engine_client.generate(
|
||||||
engine_prompt,
|
engine_input,
|
||||||
sampling_params,
|
sampling_params,
|
||||||
sub_request_id,
|
sub_request_id,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ from vllm.entrypoints.openai.engine.serving import (
|
|||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.inputs.data import ProcessorInputs
|
from vllm.inputs import EngineInput
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
from vllm.outputs import RequestOutput
|
from vllm.outputs import RequestOutput
|
||||||
@@ -82,7 +82,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
async def render_completion_request(
|
async def render_completion_request(
|
||||||
self,
|
self,
|
||||||
request: CompletionRequest,
|
request: CompletionRequest,
|
||||||
) -> list[ProcessorInputs] | ErrorResponse:
|
) -> list[EngineInput] | ErrorResponse:
|
||||||
"""
|
"""
|
||||||
Validate the model and preprocess a completion request.
|
Validate the model and preprocess a completion request.
|
||||||
|
|
||||||
@@ -90,8 +90,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
engine-aware checks (LoRA model validation, engine health).
|
engine-aware checks (LoRA model validation, engine health).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A list of engine_prompts on success,
|
A list of engine_inputs on success, or an ErrorResponse on failure.
|
||||||
or an ErrorResponse on failure.
|
|
||||||
"""
|
"""
|
||||||
error_check_ret = await self._check_model(request)
|
error_check_ret = await self._check_model(request)
|
||||||
if error_check_ret is not None:
|
if error_check_ret is not None:
|
||||||
@@ -128,7 +127,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
engine_prompts = result
|
engine_inputs = result
|
||||||
|
|
||||||
request_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
|
request_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||||
created_time = int(time.time())
|
created_time = int(time.time())
|
||||||
@@ -145,11 +144,11 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
# Schedule the request and get the result generator.
|
# Schedule the request and get the result generator.
|
||||||
max_model_len = self.model_config.max_model_len
|
max_model_len = self.model_config.max_model_len
|
||||||
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
||||||
for i, engine_prompt in enumerate(engine_prompts):
|
for i, engine_input in enumerate(engine_inputs):
|
||||||
max_tokens = get_max_tokens(
|
max_tokens = get_max_tokens(
|
||||||
max_model_len,
|
max_model_len,
|
||||||
request.max_tokens,
|
request.max_tokens,
|
||||||
self._extract_prompt_len(engine_prompt),
|
self._extract_prompt_len(engine_input),
|
||||||
self.default_sampling_params,
|
self.default_sampling_params,
|
||||||
self.override_max_tokens,
|
self.override_max_tokens,
|
||||||
)
|
)
|
||||||
@@ -169,7 +168,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id_item,
|
request_id_item,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=sampling_params,
|
params=sampling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
@@ -182,7 +181,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
|
|
||||||
if isinstance(sampling_params, BeamSearchParams):
|
if isinstance(sampling_params, BeamSearchParams):
|
||||||
generator = self.beam_search(
|
generator = self.beam_search(
|
||||||
prompt=engine_prompt,
|
prompt=engine_input,
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
params=sampling_params,
|
params=sampling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -190,7 +189,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
generator = self.engine_client.generate(
|
generator = self.engine_client.generate(
|
||||||
engine_prompt,
|
engine_input,
|
||||||
sampling_params,
|
sampling_params,
|
||||||
request_id_item,
|
request_id_item,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -204,7 +203,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
result_generator = merge_async_iterators(*generators)
|
result_generator = merge_async_iterators(*generators)
|
||||||
|
|
||||||
model_name = self.models.model_name(lora_request)
|
model_name = self.models.model_name(lora_request)
|
||||||
num_prompts = len(engine_prompts)
|
num_prompts = len(engine_inputs)
|
||||||
|
|
||||||
# Streaming response
|
# Streaming response
|
||||||
tokenizer = self.renderer.tokenizer
|
tokenizer = self.renderer.tokenizer
|
||||||
@@ -212,7 +211,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
if request.stream:
|
if request.stream:
|
||||||
return self.completion_stream_generator(
|
return self.completion_stream_generator(
|
||||||
request,
|
request,
|
||||||
engine_prompts,
|
engine_inputs,
|
||||||
result_generator,
|
result_generator,
|
||||||
request_id,
|
request_id,
|
||||||
created_time,
|
created_time,
|
||||||
@@ -235,8 +234,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
# We did not pass it into vLLM engine to avoid being redundant
|
# We did not pass it into vLLM engine to avoid being redundant
|
||||||
# with the inputs token IDs
|
# with the inputs token IDs
|
||||||
if final_res.prompt is None:
|
if final_res.prompt is None:
|
||||||
engine_prompt = engine_prompts[i]
|
final_res.prompt = self._extract_prompt_text(engine_inputs[i])
|
||||||
final_res.prompt = self._extract_prompt_text(engine_prompt)
|
|
||||||
|
|
||||||
final_res_batch_checked = cast(list[RequestOutput], final_res_batch)
|
final_res_batch_checked = cast(list[RequestOutput], final_res_batch)
|
||||||
|
|
||||||
@@ -268,7 +266,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
async def completion_stream_generator(
|
async def completion_stream_generator(
|
||||||
self,
|
self,
|
||||||
request: CompletionRequest,
|
request: CompletionRequest,
|
||||||
engine_prompts: list[ProcessorInputs],
|
engine_inputs: list[EngineInput],
|
||||||
result_generator: AsyncIterator[tuple[int, RequestOutput]],
|
result_generator: AsyncIterator[tuple[int, RequestOutput]],
|
||||||
request_id: str,
|
request_id: str,
|
||||||
created_time: int,
|
created_time: int,
|
||||||
@@ -301,8 +299,8 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
|
|
||||||
prompt_text = res.prompt
|
prompt_text = res.prompt
|
||||||
if prompt_text is None:
|
if prompt_text is None:
|
||||||
engine_prompt = engine_prompts[prompt_idx]
|
engine_input = engine_inputs[prompt_idx]
|
||||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
prompt_text = self._extract_prompt_text(engine_input)
|
||||||
|
|
||||||
# Prompt details are excluded from later streamed outputs
|
# Prompt details are excluded from later streamed outputs
|
||||||
if prompt_token_ids is not None:
|
if prompt_token_ids is not None:
|
||||||
|
|||||||
@@ -72,11 +72,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.utils import create_error_response
|
from vllm.entrypoints.utils import create_error_response
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.inputs.data import (
|
from vllm.inputs import EngineInput, PromptType, TokensPrompt
|
||||||
ProcessorInputs,
|
|
||||||
PromptType,
|
|
||||||
TokensPrompt,
|
|
||||||
)
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob, PromptLogprobs
|
from vllm.logprobs import Logprob, PromptLogprobs
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
@@ -163,7 +159,7 @@ class ServeContext(Generic[RequestT]):
|
|||||||
request_id: str
|
request_id: str
|
||||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||||
lora_request: LoRARequest | None = None
|
lora_request: LoRARequest | None = None
|
||||||
engine_prompts: list[ProcessorInputs] | None = None
|
engine_inputs: list[EngineInput] | None = None
|
||||||
|
|
||||||
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
|
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
|
||||||
None
|
None
|
||||||
@@ -202,7 +198,7 @@ class OpenAIServing:
|
|||||||
|
|
||||||
async def beam_search(
|
async def beam_search(
|
||||||
self,
|
self,
|
||||||
prompt: ProcessorInputs,
|
prompt: EngineInput,
|
||||||
request_id: str,
|
request_id: str,
|
||||||
params: BeamSearchParams,
|
params: BeamSearchParams,
|
||||||
lora_request: LoRARequest | None = None,
|
lora_request: LoRARequest | None = None,
|
||||||
@@ -493,21 +489,21 @@ class OpenAIServing:
|
|||||||
if isinstance(pooling_params, ErrorResponse):
|
if isinstance(pooling_params, ErrorResponse):
|
||||||
return pooling_params
|
return pooling_params
|
||||||
|
|
||||||
if ctx.engine_prompts is None:
|
if ctx.engine_inputs is None:
|
||||||
return self.create_error_response("Engine prompts not available")
|
return self.create_error_response("Engine prompts not available")
|
||||||
|
|
||||||
for i, engine_prompt in enumerate(ctx.engine_prompts):
|
for i, engine_input in enumerate(ctx.engine_inputs):
|
||||||
request_id_item = f"{ctx.request_id}-{i}"
|
request_id_item = f"{ctx.request_id}-{i}"
|
||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id_item,
|
request_id_item,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=pooling_params,
|
params=pooling_params,
|
||||||
lora_request=ctx.lora_request,
|
lora_request=ctx.lora_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
generator = self.engine_client.encode(
|
generator = self.engine_client.encode(
|
||||||
engine_prompt,
|
engine_input,
|
||||||
pooling_params,
|
pooling_params,
|
||||||
request_id_item,
|
request_id_item,
|
||||||
lora_request=ctx.lora_request,
|
lora_request=ctx.lora_request,
|
||||||
@@ -526,10 +522,10 @@ class OpenAIServing:
|
|||||||
ctx: ServeContext,
|
ctx: ServeContext,
|
||||||
) -> ErrorResponse | None:
|
) -> ErrorResponse | None:
|
||||||
"""Collect batch results from the result generator."""
|
"""Collect batch results from the result generator."""
|
||||||
if ctx.engine_prompts is None:
|
if ctx.engine_inputs is None:
|
||||||
return self.create_error_response("Engine prompts not available")
|
return self.create_error_response("Engine prompts not available")
|
||||||
|
|
||||||
num_prompts = len(ctx.engine_prompts)
|
num_prompts = len(ctx.engine_inputs)
|
||||||
final_res_batch: list[PoolingRequestOutput | None]
|
final_res_batch: list[PoolingRequestOutput | None]
|
||||||
final_res_batch = [None] * num_prompts
|
final_res_batch = [None] * num_prompts
|
||||||
|
|
||||||
@@ -806,19 +802,19 @@ class OpenAIServing:
|
|||||||
# Apply server defaults first, then request kwargs override.
|
# Apply server defaults first, then request kwargs override.
|
||||||
return default_chat_template_kwargs | request_chat_template_kwargs
|
return default_chat_template_kwargs | request_chat_template_kwargs
|
||||||
|
|
||||||
def _extract_prompt_components(self, prompt: PromptType | ProcessorInputs):
|
def _extract_prompt_components(self, prompt: PromptType | EngineInput):
|
||||||
return extract_prompt_components(self.model_config, prompt)
|
return extract_prompt_components(self.model_config, prompt)
|
||||||
|
|
||||||
def _extract_prompt_text(self, prompt: ProcessorInputs):
|
def _extract_prompt_text(self, prompt: PromptType | EngineInput):
|
||||||
return self._extract_prompt_components(prompt).text
|
return self._extract_prompt_components(prompt).text
|
||||||
|
|
||||||
def _extract_prompt_len(self, prompt: ProcessorInputs):
|
def _extract_prompt_len(self, prompt: EngineInput):
|
||||||
return extract_prompt_len(self.model_config, prompt)
|
return extract_prompt_len(self.model_config, prompt)
|
||||||
|
|
||||||
def _log_inputs(
|
def _log_inputs(
|
||||||
self,
|
self,
|
||||||
request_id: str,
|
request_id: str,
|
||||||
inputs: PromptType | ProcessorInputs,
|
inputs: PromptType | EngineInput,
|
||||||
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
||||||
lora_request: LoRARequest | None,
|
lora_request: LoRARequest | None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from vllm.engine.protocol import EngineClient, StreamingInput
|
|||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.inputs.data import PromptType
|
from vllm.inputs import PromptType
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.models.interfaces import SupportsRealtime
|
from vllm.model_executor.models.interfaces import SupportsRealtime
|
||||||
from vllm.renderers.inputs.preprocess import parse_model_prompt
|
from vllm.renderers.inputs.preprocess import parse_model_prompt
|
||||||
@@ -83,6 +83,6 @@ class OpenAIServingRealtime(OpenAIServing):
|
|||||||
|
|
||||||
async for prompt in stream_input_iter:
|
async for prompt in stream_input_iter:
|
||||||
parsed_prompt = parse_model_prompt(model_config, prompt)
|
parsed_prompt = parse_model_prompt(model_config, prompt)
|
||||||
(engine_prompt,) = await renderer.render_cmpl_async([parsed_prompt])
|
(engine_input,) = await renderer.render_cmpl_async([parsed_prompt])
|
||||||
|
|
||||||
yield StreamingInput(prompt=engine_prompt)
|
yield StreamingInput(prompt=engine_input)
|
||||||
|
|||||||
@@ -110,7 +110,7 @@ from vllm.entrypoints.openai.responses.utils import (
|
|||||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||||
from vllm.entrypoints.utils import get_max_tokens
|
from vllm.entrypoints.utils import get_max_tokens
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.inputs.data import ProcessorInputs, token_inputs
|
from vllm.inputs import EngineInput, tokens_input
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob as SampleLogprob
|
from vllm.logprobs import Logprob as SampleLogprob
|
||||||
from vllm.logprobs import SampleLogprobs
|
from vllm.logprobs import SampleLogprobs
|
||||||
@@ -269,10 +269,10 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
|
|
||||||
def _validate_generator_input(
|
def _validate_generator_input(
|
||||||
self,
|
self,
|
||||||
engine_prompt: ProcessorInputs,
|
engine_input: EngineInput,
|
||||||
) -> ErrorResponse | None:
|
) -> ErrorResponse | None:
|
||||||
"""Add validations to the input to the generator here."""
|
"""Add validations to the input to the generator here."""
|
||||||
prompt_len = self._extract_prompt_len(engine_prompt)
|
prompt_len = self._extract_prompt_len(engine_input)
|
||||||
max_model_len = self.model_config.max_model_len
|
max_model_len = self.model_config.max_model_len
|
||||||
|
|
||||||
if prompt_len >= max_model_len:
|
if prompt_len >= max_model_len:
|
||||||
@@ -369,11 +369,11 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
model_name = self.models.model_name(lora_request)
|
model_name = self.models.model_name(lora_request)
|
||||||
|
|
||||||
if self.use_harmony:
|
if self.use_harmony:
|
||||||
messages, engine_prompts = self._make_request_with_harmony(
|
messages, engine_inputs = self._make_request_with_harmony(
|
||||||
request, prev_response
|
request, prev_response
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
messages, engine_prompts = await self._make_request(request, prev_response)
|
messages, engine_inputs = await self._make_request(request, prev_response)
|
||||||
|
|
||||||
request_metadata = RequestResponseMetadata(request_id=request.request_id)
|
request_metadata = RequestResponseMetadata(request_id=request.request_id)
|
||||||
if raw_request:
|
if raw_request:
|
||||||
@@ -413,15 +413,15 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
available_tools = []
|
available_tools = []
|
||||||
tokenizer = self.renderer.get_tokenizer()
|
tokenizer = self.renderer.get_tokenizer()
|
||||||
|
|
||||||
for engine_prompt in engine_prompts:
|
for engine_input in engine_inputs:
|
||||||
maybe_error = self._validate_generator_input(engine_prompt)
|
maybe_error = self._validate_generator_input(engine_input)
|
||||||
if maybe_error is not None:
|
if maybe_error is not None:
|
||||||
return maybe_error
|
return maybe_error
|
||||||
|
|
||||||
default_max_tokens = get_max_tokens(
|
default_max_tokens = get_max_tokens(
|
||||||
max_model_len,
|
max_model_len,
|
||||||
request.max_output_tokens,
|
request.max_output_tokens,
|
||||||
self._extract_prompt_len(engine_prompt),
|
self._extract_prompt_len(engine_input),
|
||||||
self.default_sampling_params,
|
self.default_sampling_params,
|
||||||
self.override_max_tokens,
|
self.override_max_tokens,
|
||||||
)
|
)
|
||||||
@@ -480,7 +480,7 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
)
|
)
|
||||||
generator = self._generate_with_builtin_tools(
|
generator = self._generate_with_builtin_tools(
|
||||||
request_id=request.request_id,
|
request_id=request.request_id,
|
||||||
engine_prompt=engine_prompt,
|
engine_input=engine_input,
|
||||||
sampling_params=sampling_params,
|
sampling_params=sampling_params,
|
||||||
context=context,
|
context=context,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -586,7 +586,7 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
prev_response_output=prev_response.output if prev_response else None,
|
prev_response_output=prev_response.output if prev_response else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
_, engine_prompts = await self.openai_serving_render.preprocess_chat(
|
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||||
request,
|
request,
|
||||||
messages,
|
messages,
|
||||||
default_template=self.chat_template,
|
default_template=self.chat_template,
|
||||||
@@ -595,7 +595,7 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
tool_dicts=tool_dicts,
|
tool_dicts=tool_dicts,
|
||||||
tool_parser=self.parser.tool_parser_cls if self.parser else None,
|
tool_parser=self.parser.tool_parser_cls if self.parser else None,
|
||||||
)
|
)
|
||||||
return messages, engine_prompts
|
return messages, engine_inputs
|
||||||
|
|
||||||
async def _render_next_turn(
|
async def _render_next_turn(
|
||||||
self,
|
self,
|
||||||
@@ -610,7 +610,7 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
request_input=messages,
|
request_input=messages,
|
||||||
)
|
)
|
||||||
|
|
||||||
_, engine_prompts = await self.openai_serving_render.preprocess_chat(
|
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||||
request,
|
request,
|
||||||
new_messages,
|
new_messages,
|
||||||
default_template=chat_template,
|
default_template=chat_template,
|
||||||
@@ -619,12 +619,12 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
tool_dicts=tool_dicts,
|
tool_dicts=tool_dicts,
|
||||||
tool_parser=tool_parser,
|
tool_parser=tool_parser,
|
||||||
)
|
)
|
||||||
return engine_prompts
|
return engine_inputs
|
||||||
|
|
||||||
async def _generate_with_builtin_tools(
|
async def _generate_with_builtin_tools(
|
||||||
self,
|
self,
|
||||||
request_id: str,
|
request_id: str,
|
||||||
engine_prompt: ProcessorInputs,
|
engine_input: EngineInput,
|
||||||
sampling_params: SamplingParams,
|
sampling_params: SamplingParams,
|
||||||
context: ConversationContext,
|
context: ConversationContext,
|
||||||
lora_request: LoRARequest | None = None,
|
lora_request: LoRARequest | None = None,
|
||||||
@@ -641,13 +641,13 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
sub_request_id,
|
sub_request_id,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=sampling_params,
|
params=sampling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
generator = self.engine_client.generate(
|
generator = self.engine_client.generate(
|
||||||
engine_prompt,
|
engine_input,
|
||||||
sampling_params,
|
sampling_params,
|
||||||
sub_request_id,
|
sub_request_id,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -675,11 +675,11 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
# Render the next prompt token ids and update sampling_params.
|
# Render the next prompt token ids and update sampling_params.
|
||||||
if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
|
if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
|
||||||
token_ids = context.render_for_completion()
|
token_ids = context.render_for_completion()
|
||||||
engine_prompt = token_inputs(token_ids)
|
engine_input = tokens_input(token_ids)
|
||||||
|
|
||||||
sampling_params.max_tokens = max_model_len - len(token_ids)
|
sampling_params.max_tokens = max_model_len - len(token_ids)
|
||||||
elif isinstance(context, ParsableContext):
|
elif isinstance(context, ParsableContext):
|
||||||
(engine_prompt,) = await self._render_next_turn(
|
(engine_input,) = await self._render_next_turn(
|
||||||
context.request,
|
context.request,
|
||||||
context.parser.response_messages,
|
context.parser.response_messages,
|
||||||
context.tool_dicts,
|
context.tool_dicts,
|
||||||
@@ -691,7 +691,7 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
sampling_params.max_tokens = get_max_tokens(
|
sampling_params.max_tokens = get_max_tokens(
|
||||||
max_model_len,
|
max_model_len,
|
||||||
context.request.max_output_tokens,
|
context.request.max_output_tokens,
|
||||||
self._extract_prompt_len(engine_prompt),
|
self._extract_prompt_len(engine_input),
|
||||||
self.default_sampling_params, # type: ignore
|
self.default_sampling_params, # type: ignore
|
||||||
self.override_max_tokens, # type: ignore
|
self.override_max_tokens, # type: ignore
|
||||||
)
|
)
|
||||||
@@ -713,14 +713,10 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
arrival_time = time.time()
|
arrival_time = time.time()
|
||||||
messages = self._construct_input_messages_with_harmony(request, prev_response)
|
messages = self._construct_input_messages_with_harmony(request, prev_response)
|
||||||
prompt_token_ids = render_for_completion(messages)
|
prompt_token_ids = render_for_completion(messages)
|
||||||
engine_prompt = token_inputs(prompt_token_ids)
|
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
|
||||||
engine_prompt["arrival_time"] = arrival_time
|
engine_input["arrival_time"] = arrival_time
|
||||||
|
|
||||||
# Add cache_salt if provided in the request
|
return messages, [engine_input]
|
||||||
if request.cache_salt is not None:
|
|
||||||
engine_prompt["cache_salt"] = request.cache_salt
|
|
||||||
|
|
||||||
return messages, [engine_prompt]
|
|
||||||
|
|
||||||
async def _initialize_tool_sessions(
|
async def _initialize_tool_sessions(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ from vllm.entrypoints.openai.speech_to_text.protocol import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.utils import get_max_tokens
|
from vllm.entrypoints.utils import get_max_tokens
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.inputs import EncoderDecoderInputs, ProcessorInputs
|
from vllm.inputs import EncoderDecoderInput, EngineInput
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import FlatLogprobs, Logprob
|
from vllm.logprobs import FlatLogprobs, Logprob
|
||||||
from vllm.model_executor.models import SupportsTranscription
|
from vllm.model_executor.models import SupportsTranscription
|
||||||
@@ -171,7 +171,7 @@ class OpenAISpeechToText(OpenAIServing):
|
|||||||
request: SpeechToTextRequest,
|
request: SpeechToTextRequest,
|
||||||
audio_data: bytes,
|
audio_data: bytes,
|
||||||
request_id: str,
|
request_id: str,
|
||||||
) -> tuple[list[ProcessorInputs], float]:
|
) -> tuple[list[EngineInput], float]:
|
||||||
# Validate request
|
# Validate request
|
||||||
language = self.model_cls.validate_language(request.language)
|
language = self.model_cls.validate_language(request.language)
|
||||||
# Skip to_language validation to avoid extra logging for Whisper.
|
# Skip to_language validation to avoid extra logging for Whisper.
|
||||||
@@ -250,9 +250,9 @@ class OpenAISpeechToText(OpenAIServing):
|
|||||||
|
|
||||||
parsed_prompts.append(parsed_prompt)
|
parsed_prompts.append(parsed_prompt)
|
||||||
|
|
||||||
engine_prompts = await self.renderer.render_cmpl_async(parsed_prompts)
|
engine_inputs = await self.renderer.render_cmpl_async(parsed_prompts)
|
||||||
|
|
||||||
return engine_prompts, duration
|
return engine_inputs, duration
|
||||||
|
|
||||||
def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):
|
def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):
|
||||||
dec_prompt = prompt["decoder_prompt"]
|
dec_prompt = prompt["decoder_prompt"]
|
||||||
@@ -271,7 +271,7 @@ class OpenAISpeechToText(OpenAIServing):
|
|||||||
return prompt
|
return prompt
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_decoder_prompt_len(engine_prompts: list[ProcessorInputs]) -> int:
|
def _get_decoder_prompt_len(engine_inputs: list[EngineInput]) -> int:
|
||||||
"""Get the length of the decoder prompt. Currently we need to offset
|
"""Get the length of the decoder prompt. Currently we need to offset
|
||||||
by the decoder prompt length when running beam search because the mm
|
by the decoder prompt length when running beam search because the mm
|
||||||
encoder is not currently cached and runs on decode calls; because of
|
encoder is not currently cached and runs on decode calls; because of
|
||||||
@@ -282,12 +282,13 @@ class OpenAISpeechToText(OpenAIServing):
|
|||||||
encoder/decoder caching is implemented.
|
encoder/decoder caching is implemented.
|
||||||
"""
|
"""
|
||||||
input_len = 0
|
input_len = 0
|
||||||
assert len(engine_prompts) > 0
|
assert len(engine_inputs) > 0
|
||||||
first_eng_prompt = engine_prompts[0]
|
first_input = engine_inputs[0]
|
||||||
|
|
||||||
|
if first_input.get("type") == "enc_dec":
|
||||||
|
first_input = cast(EncoderDecoderInput, first_input)
|
||||||
|
input_len = len(first_input["decoder_prompt"]["prompt_token_ids"])
|
||||||
|
|
||||||
if first_eng_prompt.get("type") == "enc_dec":
|
|
||||||
first_eng_prompt = cast(EncoderDecoderInputs, first_eng_prompt)
|
|
||||||
input_len = len(first_eng_prompt["decoder_prompt"]["prompt_token_ids"])
|
|
||||||
return input_len
|
return input_len
|
||||||
|
|
||||||
def _get_verbose_segments(
|
def _get_verbose_segments(
|
||||||
@@ -409,7 +410,7 @@ class OpenAISpeechToText(OpenAIServing):
|
|||||||
|
|
||||||
lora_request = self._maybe_get_adapters(request)
|
lora_request = self._maybe_get_adapters(request)
|
||||||
|
|
||||||
engine_prompts, duration_s = await self._preprocess_speech_to_text(
|
engine_inputs, duration_s = await self._preprocess_speech_to_text(
|
||||||
request=request,
|
request=request,
|
||||||
audio_data=audio_data,
|
audio_data=audio_data,
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
@@ -420,7 +421,7 @@ class OpenAISpeechToText(OpenAIServing):
|
|||||||
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
|
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
|
||||||
|
|
||||||
input_len = (
|
input_len = (
|
||||||
OpenAISpeechToText._get_decoder_prompt_len(engine_prompts)
|
OpenAISpeechToText._get_decoder_prompt_len(engine_inputs)
|
||||||
if request.use_beam_search
|
if request.use_beam_search
|
||||||
else 0
|
else 0
|
||||||
)
|
)
|
||||||
@@ -450,12 +451,12 @@ class OpenAISpeechToText(OpenAIServing):
|
|||||||
sampling_params.logprobs = 1
|
sampling_params.logprobs = 1
|
||||||
|
|
||||||
list_result_generator = []
|
list_result_generator = []
|
||||||
for i, engine_prompt in enumerate(engine_prompts):
|
for i, engine_input in enumerate(engine_inputs):
|
||||||
request_id_item = f"{request_id}_{i}"
|
request_id_item = f"{request_id}_{i}"
|
||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id_item,
|
request_id_item,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=sampling_params,
|
params=sampling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
@@ -468,7 +469,7 @@ class OpenAISpeechToText(OpenAIServing):
|
|||||||
|
|
||||||
if isinstance(sampling_params, BeamSearchParams):
|
if isinstance(sampling_params, BeamSearchParams):
|
||||||
generator = self.beam_search(
|
generator = self.beam_search(
|
||||||
prompt=engine_prompt,
|
prompt=engine_input,
|
||||||
params=sampling_params,
|
params=sampling_params,
|
||||||
request_id=request_id_item,
|
request_id=request_id_item,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -476,7 +477,7 @@ class OpenAISpeechToText(OpenAIServing):
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
generator = self.engine_client.generate(
|
generator = self.engine_client.generate(
|
||||||
engine_prompt,
|
engine_input,
|
||||||
sampling_params,
|
sampling_params,
|
||||||
request_id_item,
|
request_id_item,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from vllm.entrypoints.pooling.typing import (
|
|||||||
PoolingCompletionLikeRequest,
|
PoolingCompletionLikeRequest,
|
||||||
PoolingServeContext,
|
PoolingServeContext,
|
||||||
)
|
)
|
||||||
from vllm.inputs.data import ProcessorInputs, SingletonPrompt
|
from vllm.inputs import EngineInput, SingletonPrompt
|
||||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||||
from vllm.tool_parsers import ToolParser
|
from vllm.tool_parsers import ToolParser
|
||||||
@@ -60,7 +60,7 @@ class PoolingIOProcessor:
|
|||||||
chat_template_kwargs=request.chat_template_kwargs,
|
chat_template_kwargs=request.chat_template_kwargs,
|
||||||
trust_request_chat_template=self.trust_request_chat_template,
|
trust_request_chat_template=self.trust_request_chat_template,
|
||||||
)
|
)
|
||||||
_, engine_prompts = self._preprocess_chat_online(
|
_, engine_inputs = self._preprocess_chat_online(
|
||||||
request,
|
request,
|
||||||
request.messages,
|
request.messages,
|
||||||
default_template=self.chat_template,
|
default_template=self.chat_template,
|
||||||
@@ -68,7 +68,7 @@ class PoolingIOProcessor:
|
|||||||
default_template_kwargs=None,
|
default_template_kwargs=None,
|
||||||
)
|
)
|
||||||
elif isinstance(request, PoolingCompletionLikeRequest):
|
elif isinstance(request, PoolingCompletionLikeRequest):
|
||||||
engine_prompts = self._preprocess_completion_online(
|
engine_inputs = self._preprocess_completion_online(
|
||||||
request,
|
request,
|
||||||
prompt_input=request.input,
|
prompt_input=request.input,
|
||||||
prompt_embeds=None,
|
prompt_embeds=None,
|
||||||
@@ -76,7 +76,7 @@ class PoolingIOProcessor:
|
|||||||
else:
|
else:
|
||||||
raise ValueError(f"Invalid {self.name} request type")
|
raise ValueError(f"Invalid {self.name} request type")
|
||||||
|
|
||||||
ctx.engine_prompts = engine_prompts
|
ctx.engine_inputs = engine_inputs
|
||||||
|
|
||||||
async def pre_process_online_async(self, ctx: PoolingServeContext):
|
async def pre_process_online_async(self, ctx: PoolingServeContext):
|
||||||
self.pre_process_online(ctx)
|
self.pre_process_online(ctx)
|
||||||
@@ -100,7 +100,7 @@ class PoolingIOProcessor:
|
|||||||
self,
|
self,
|
||||||
prompts: PromptType | Sequence[PromptType],
|
prompts: PromptType | Sequence[PromptType],
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> Sequence[ProcessorInputs]:
|
) -> Sequence[EngineInput]:
|
||||||
return self._preprocess_completion_offline(
|
return self._preprocess_completion_offline(
|
||||||
prompts=prompts, tokenization_kwargs=tokenization_kwargs
|
prompts=prompts, tokenization_kwargs=tokenization_kwargs
|
||||||
)
|
)
|
||||||
@@ -128,7 +128,7 @@ class PoolingIOProcessor:
|
|||||||
request: RendererRequest,
|
request: RendererRequest,
|
||||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||||
prompt_embeds: bytes | list[bytes] | None,
|
prompt_embeds: bytes | list[bytes] | None,
|
||||||
) -> list[ProcessorInputs]:
|
) -> list[EngineInput]:
|
||||||
renderer = self.renderer
|
renderer = self.renderer
|
||||||
model_config = self.model_config
|
model_config = self.model_config
|
||||||
|
|
||||||
@@ -167,7 +167,7 @@ class PoolingIOProcessor:
|
|||||||
default_template_kwargs: dict[str, Any] | None,
|
default_template_kwargs: dict[str, Any] | None,
|
||||||
tool_dicts: list[dict[str, Any]] | None = None,
|
tool_dicts: list[dict[str, Any]] | None = None,
|
||||||
tool_parser: type[ToolParser] | None = None,
|
tool_parser: type[ToolParser] | None = None,
|
||||||
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
|
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||||
renderer = self.renderer
|
renderer = self.renderer
|
||||||
|
|
||||||
default_template_kwargs = merge_kwargs(
|
default_template_kwargs = merge_kwargs(
|
||||||
@@ -188,7 +188,7 @@ class PoolingIOProcessor:
|
|||||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||||
)
|
)
|
||||||
|
|
||||||
(conversation,), (engine_prompt,) = renderer.render_chat(
|
(conversation,), (engine_input,) = renderer.render_chat(
|
||||||
[messages],
|
[messages],
|
||||||
chat_params,
|
chat_params,
|
||||||
tok_params,
|
tok_params,
|
||||||
@@ -199,13 +199,13 @@ class PoolingIOProcessor:
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
return conversation, [engine_prompt]
|
return conversation, [engine_input]
|
||||||
|
|
||||||
def _preprocess_completion_offline(
|
def _preprocess_completion_offline(
|
||||||
self,
|
self,
|
||||||
prompts: PromptType | Sequence[PromptType],
|
prompts: PromptType | Sequence[PromptType],
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> Sequence[ProcessorInputs]:
|
) -> Sequence[EngineInput]:
|
||||||
renderer = self.renderer
|
renderer = self.renderer
|
||||||
model_config = self.model_config
|
model_config = self.model_config
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.pooling.typing import AnyPoolingRequest, PoolingServeContext
|
from vllm.entrypoints.pooling.typing import AnyPoolingRequest, PoolingServeContext
|
||||||
from vllm.exceptions import VLLMNotFoundError
|
from vllm.exceptions import VLLMNotFoundError
|
||||||
from vllm.inputs.data import ProcessorInputs
|
from vllm.inputs import EngineInput
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
from vllm.renderers.base import BaseRenderer
|
from vllm.renderers.base import BaseRenderer
|
||||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||||
@@ -106,7 +106,7 @@ class PoolingServing:
|
|||||||
self,
|
self,
|
||||||
ctx: PoolingServeContext,
|
ctx: PoolingServeContext,
|
||||||
):
|
):
|
||||||
if ctx.engine_prompts is None:
|
if ctx.engine_inputs is None:
|
||||||
raise ValueError("Engine prompts not available")
|
raise ValueError("Engine prompts not available")
|
||||||
|
|
||||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||||
@@ -120,7 +120,7 @@ class PoolingServing:
|
|||||||
pooling_params = self.io_processor.create_pooling_params(ctx.request)
|
pooling_params = self.io_processor.create_pooling_params(ctx.request)
|
||||||
pooling_params.verify(self.model_config)
|
pooling_params.verify(self.model_config)
|
||||||
|
|
||||||
for i, engine_prompt in enumerate(ctx.engine_prompts):
|
for i, engine_input in enumerate(ctx.engine_inputs):
|
||||||
prompt_request_id = (
|
prompt_request_id = (
|
||||||
f"{ctx.request_id}-{i}"
|
f"{ctx.request_id}-{i}"
|
||||||
if ctx.prompt_request_ids is None
|
if ctx.prompt_request_ids is None
|
||||||
@@ -129,13 +129,13 @@ class PoolingServing:
|
|||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
prompt_request_id,
|
prompt_request_id,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=pooling_params,
|
params=pooling_params,
|
||||||
lora_request=ctx.lora_request,
|
lora_request=ctx.lora_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
generator = self.engine_client.encode(
|
generator = self.engine_client.encode(
|
||||||
engine_prompt,
|
engine_input,
|
||||||
pooling_params,
|
pooling_params,
|
||||||
prompt_request_id,
|
prompt_request_id,
|
||||||
lora_request=ctx.lora_request,
|
lora_request=ctx.lora_request,
|
||||||
@@ -151,13 +151,13 @@ class PoolingServing:
|
|||||||
self,
|
self,
|
||||||
ctx: PoolingServeContext,
|
ctx: PoolingServeContext,
|
||||||
):
|
):
|
||||||
if ctx.engine_prompts is None:
|
if ctx.engine_inputs is None:
|
||||||
raise ValueError("Engine prompts not available")
|
raise ValueError("Engine prompts not available")
|
||||||
|
|
||||||
if ctx.result_generator is None:
|
if ctx.result_generator is None:
|
||||||
raise ValueError("Result generator not available")
|
raise ValueError("Result generator not available")
|
||||||
|
|
||||||
num_inputs = len(ctx.engine_prompts)
|
num_inputs = len(ctx.engine_inputs)
|
||||||
final_res_batch: list[PoolingRequestOutput | None]
|
final_res_batch: list[PoolingRequestOutput | None]
|
||||||
final_res_batch = [None] * num_inputs
|
final_res_batch = [None] * num_inputs
|
||||||
|
|
||||||
@@ -317,7 +317,7 @@ class PoolingServing:
|
|||||||
def _log_inputs(
|
def _log_inputs(
|
||||||
self,
|
self,
|
||||||
request_id: str,
|
request_id: str,
|
||||||
inputs: ProcessorInputs,
|
inputs: EngineInput,
|
||||||
params: PoolingParams,
|
params: PoolingParams,
|
||||||
lora_request: LoRARequest | None,
|
lora_request: LoRARequest | None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from vllm.entrypoints.pooling.embed.protocol import (
|
|||||||
EmbeddingCompletionRequest,
|
EmbeddingCompletionRequest,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||||
from vllm.inputs.data import ProcessorInputs, token_inputs
|
from vllm.inputs import EngineInput, tokens_input
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||||
from vllm.renderers import merge_kwargs
|
from vllm.renderers import merge_kwargs
|
||||||
@@ -83,20 +83,20 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
#################################################################
|
#################################################################
|
||||||
|
|
||||||
def _pre_process_chunked(self, ctx: PoolingServeContext) -> None:
|
def _pre_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||||
if ctx.engine_prompts is None:
|
if ctx.engine_inputs is None:
|
||||||
raise ValueError("Engine prompts not available")
|
raise ValueError("Engine prompts not available")
|
||||||
|
|
||||||
ctx.intermediates = ctx.engine_prompts
|
ctx.intermediates = ctx.engine_inputs
|
||||||
request_id = ctx.request_id
|
request_id = ctx.request_id
|
||||||
max_model_len = self.model_config.max_model_len
|
max_model_len = self.model_config.max_model_len
|
||||||
chunked_engine_prompts: list[ProcessorInputs] = []
|
chunked_engine_inputs: list[EngineInput] = []
|
||||||
prompt_request_ids: list[str] = []
|
prompt_request_ids: list[str] = []
|
||||||
for prompt_idx, engine_prompt in enumerate(ctx.engine_prompts):
|
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
|
||||||
token_ids = engine_prompt.get("prompt_token_ids", None)
|
token_ids = engine_input.get("prompt_token_ids", None)
|
||||||
if token_ids is None:
|
if token_ids is None:
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Long Text Embedding with Chunked Processing does "
|
"Long Text Embedding with Chunked Processing does "
|
||||||
"not support EmbedsPrompt and EncoderDecoderInputs."
|
"not support EmbedsPrompt and EncoderDecoderInput."
|
||||||
)
|
)
|
||||||
|
|
||||||
prompt_token_ids = cast(list[int], token_ids)
|
prompt_token_ids = cast(list[int], token_ids)
|
||||||
@@ -104,14 +104,14 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
for chunk_idx, chunk_tokens in enumerate(
|
for chunk_idx, chunk_tokens in enumerate(
|
||||||
chunk_list(prompt_token_ids, max_model_len)
|
chunk_list(prompt_token_ids, max_model_len)
|
||||||
):
|
):
|
||||||
chunked_engine_prompts.append(
|
chunked_engine_inputs.append(
|
||||||
token_inputs(prompt_token_ids=chunk_tokens)
|
tokens_input(prompt_token_ids=chunk_tokens)
|
||||||
)
|
)
|
||||||
prompt_request_ids.append(
|
prompt_request_ids.append(
|
||||||
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
||||||
)
|
)
|
||||||
|
|
||||||
ctx.engine_prompts = chunked_engine_prompts
|
ctx.engine_inputs = chunked_engine_inputs
|
||||||
ctx.prompt_request_ids = prompt_request_ids
|
ctx.prompt_request_ids = prompt_request_ids
|
||||||
|
|
||||||
return None
|
return None
|
||||||
@@ -184,8 +184,8 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
if ctx.intermediates is None:
|
if ctx.intermediates is None:
|
||||||
raise ValueError("Original prompts inputs not available")
|
raise ValueError("Original prompts inputs not available")
|
||||||
|
|
||||||
original_engine_prompts = cast(list[ProcessorInputs], ctx.intermediates)
|
original_engine_inputs = cast(list[EngineInput], ctx.intermediates)
|
||||||
num_prompts = len(original_engine_prompts)
|
num_prompts = len(original_engine_inputs)
|
||||||
|
|
||||||
# Finalize aggregated results
|
# Finalize aggregated results
|
||||||
final_res_batch: list[PoolingRequestOutput] = []
|
final_res_batch: list[PoolingRequestOutput] = []
|
||||||
@@ -211,12 +211,12 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
pooling_output_data = PoolingOutput(data=final_embedding)
|
pooling_output_data = PoolingOutput(data=final_embedding)
|
||||||
|
|
||||||
# Get original prompt token IDs for this prompt
|
# Get original prompt token IDs for this prompt
|
||||||
original_prompt = original_engine_prompts[prompt_idx]
|
original_prompt = original_engine_inputs[prompt_idx]
|
||||||
token_ids = original_prompt.get("prompt_token_ids", None)
|
token_ids = original_prompt.get("prompt_token_ids", None)
|
||||||
if token_ids is None:
|
if token_ids is None:
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Long Text Embedding with Chunked Processing does "
|
"Long Text Embedding with Chunked Processing does "
|
||||||
"not support EmbedsPrompt and EncoderDecoderInputs."
|
"not support EmbedsPrompt and EncoderDecoderInput."
|
||||||
)
|
)
|
||||||
|
|
||||||
original_token_ids = cast(list[int], token_ids)
|
original_token_ids = cast(list[int], token_ids)
|
||||||
@@ -372,7 +372,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
]
|
]
|
||||||
for uri in request.images
|
for uri in request.images
|
||||||
]
|
]
|
||||||
ctx.engine_prompts = self._batch_render_chat(
|
ctx.engine_inputs = self._batch_render_chat(
|
||||||
request, all_messages, truncate_prompt_tokens, truncation_side
|
request, all_messages, truncate_prompt_tokens, truncation_side
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -382,7 +382,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
self._mixed_input_to_messages(inp, task_prefix=task_prefix)
|
self._mixed_input_to_messages(inp, task_prefix=task_prefix)
|
||||||
for inp in request.inputs
|
for inp in request.inputs
|
||||||
]
|
]
|
||||||
ctx.engine_prompts = self._batch_render_chat(
|
ctx.engine_inputs = self._batch_render_chat(
|
||||||
request, all_messages, truncate_prompt_tokens, truncation_side
|
request, all_messages, truncate_prompt_tokens, truncation_side
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -396,7 +396,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||||
truncation_side=truncation_side,
|
truncation_side=truncation_side,
|
||||||
)
|
)
|
||||||
ctx.engine_prompts = self._preprocess_completion_online(
|
ctx.engine_inputs = self._preprocess_completion_online(
|
||||||
proxy, prompt_input=proxy.input, prompt_embeds=None
|
proxy, prompt_input=proxy.input, prompt_embeds=None
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -406,7 +406,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
all_messages: Sequence[list[ChatCompletionMessageParam]],
|
all_messages: Sequence[list[ChatCompletionMessageParam]],
|
||||||
truncate_prompt_tokens: int | None,
|
truncate_prompt_tokens: int | None,
|
||||||
truncation_side: Literal["left", "right"] | None,
|
truncation_side: Literal["left", "right"] | None,
|
||||||
) -> list[ProcessorInputs]:
|
) -> list[EngineInput]:
|
||||||
"""Batch-render multiple conversations through the chat template."""
|
"""Batch-render multiple conversations through the chat template."""
|
||||||
if not all_messages:
|
if not all_messages:
|
||||||
return []
|
return []
|
||||||
@@ -438,8 +438,8 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||||
)
|
)
|
||||||
|
|
||||||
_, engine_prompts = renderer.render_chat(all_messages, chat_params, tok_params)
|
_, engine_inputs = renderer.render_chat(all_messages, chat_params, tok_params)
|
||||||
return engine_prompts
|
return engine_inputs
|
||||||
|
|
||||||
def _validate_input_type(self, input_type: str | None) -> None:
|
def _validate_input_type(self, input_type: str | None) -> None:
|
||||||
"""Raise if *input_type* is not supported by this model."""
|
"""Raise if *input_type* is not supported by this model."""
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ from vllm.entrypoints.pooling.utils import (
|
|||||||
encode_pooling_output_float,
|
encode_pooling_output_float,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||||
from vllm.inputs import ProcessorInputs
|
from vllm.inputs import EngineInput
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.outputs import PoolingRequestOutput
|
from vllm.outputs import PoolingRequestOutput
|
||||||
from vllm.renderers.inputs.preprocess import prompt_to_seq
|
from vllm.renderers.inputs.preprocess import prompt_to_seq
|
||||||
@@ -110,7 +110,7 @@ class OpenAIServingPooling(OpenAIServing):
|
|||||||
request.task,
|
request.task,
|
||||||
)
|
)
|
||||||
|
|
||||||
engine_prompts: Sequence[ProcessorInputs]
|
engine_inputs: Sequence[EngineInput]
|
||||||
if use_io_processor := isinstance(request, IOProcessorRequest):
|
if use_io_processor := isinstance(request, IOProcessorRequest):
|
||||||
if self.io_processor is None:
|
if self.io_processor is None:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -125,7 +125,7 @@ class OpenAIServingPooling(OpenAIServing):
|
|||||||
raw_prompts = await self.io_processor.pre_process_async(
|
raw_prompts = await self.io_processor.pre_process_async(
|
||||||
prompt=validated_prompt, request_id=request_id
|
prompt=validated_prompt, request_id=request_id
|
||||||
)
|
)
|
||||||
engine_prompts = await self.openai_serving_render.preprocess_cmpl(
|
engine_inputs = await self.openai_serving_render.preprocess_cmpl(
|
||||||
request,
|
request,
|
||||||
prompt_to_seq(raw_prompts),
|
prompt_to_seq(raw_prompts),
|
||||||
)
|
)
|
||||||
@@ -138,7 +138,7 @@ class OpenAIServingPooling(OpenAIServing):
|
|||||||
if error_check_ret is not None:
|
if error_check_ret is not None:
|
||||||
return error_check_ret
|
return error_check_ret
|
||||||
|
|
||||||
_, engine_prompts = await self.openai_serving_render.preprocess_chat(
|
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||||
request,
|
request,
|
||||||
request.messages,
|
request.messages,
|
||||||
default_template=self.chat_template,
|
default_template=self.chat_template,
|
||||||
@@ -146,7 +146,7 @@ class OpenAIServingPooling(OpenAIServing):
|
|||||||
default_template_kwargs=None,
|
default_template_kwargs=None,
|
||||||
)
|
)
|
||||||
elif isinstance(request, PoolingCompletionRequest):
|
elif isinstance(request, PoolingCompletionRequest):
|
||||||
engine_prompts = await self.openai_serving_render.preprocess_completion(
|
engine_inputs = await self.openai_serving_render.preprocess_completion(
|
||||||
request,
|
request,
|
||||||
prompt_input=request.input,
|
prompt_input=request.input,
|
||||||
prompt_embeds=None,
|
prompt_embeds=None,
|
||||||
@@ -165,12 +165,12 @@ class OpenAIServingPooling(OpenAIServing):
|
|||||||
else:
|
else:
|
||||||
pooling_params = request.to_pooling_params() # type: ignore
|
pooling_params = request.to_pooling_params() # type: ignore
|
||||||
|
|
||||||
for i, engine_prompt in enumerate(engine_prompts):
|
for i, engine_input in enumerate(engine_inputs):
|
||||||
request_id_item = f"{request_id}-{i}"
|
request_id_item = f"{request_id}-{i}"
|
||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id_item,
|
request_id_item,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=pooling_params,
|
params=pooling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
@@ -182,7 +182,7 @@ class OpenAIServingPooling(OpenAIServing):
|
|||||||
)
|
)
|
||||||
|
|
||||||
generator = self.engine_client.encode(
|
generator = self.engine_client.encode(
|
||||||
engine_prompt,
|
engine_input,
|
||||||
pooling_params,
|
pooling_params,
|
||||||
request_id_item,
|
request_id_item,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -221,7 +221,7 @@ class OpenAIServingPooling(OpenAIServing):
|
|||||||
return IOProcessorResponse(request_id=request_id, data=output)
|
return IOProcessorResponse(request_id=request_id, data=output)
|
||||||
|
|
||||||
assert isinstance(request, (PoolingCompletionRequest, PoolingChatRequest))
|
assert isinstance(request, (PoolingCompletionRequest, PoolingChatRequest))
|
||||||
num_prompts = len(engine_prompts)
|
num_prompts = len(engine_inputs)
|
||||||
|
|
||||||
# Non-streaming response
|
# Non-streaming response
|
||||||
final_res_batch: list[PoolingRequestOutput | None]
|
final_res_batch: list[PoolingRequestOutput | None]
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ from vllm.entrypoints.pooling.score.utils import (
|
|||||||
parse_score_data_single,
|
parse_score_data_single,
|
||||||
validate_score_input,
|
validate_score_input,
|
||||||
)
|
)
|
||||||
from vllm.inputs.data import ProcessorInputs, TokensPrompt, token_inputs
|
from vllm.inputs import EngineInput, TokensPrompt, tokens_input
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
|
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
|
||||||
@@ -110,12 +110,12 @@ class ServingScores(OpenAIServing):
|
|||||||
*(encode_async(t, **tokenization_kwargs) for t in input_texts)
|
*(encode_async(t, **tokenization_kwargs) for t in input_texts)
|
||||||
)
|
)
|
||||||
|
|
||||||
engine_prompts: list[ProcessorInputs] = []
|
engine_inputs: list[EngineInput] = []
|
||||||
for tok_result, input_text in zip(tokenized_prompts, input_texts):
|
for tok_result, input_text in zip(tokenized_prompts, input_texts):
|
||||||
text_token_prompt = self._validate_input(request, tok_result, input_text)
|
text_token_prompt = self._validate_input(request, tok_result, input_text)
|
||||||
|
|
||||||
engine_prompts.append(
|
engine_inputs.append(
|
||||||
token_inputs(
|
tokens_input(
|
||||||
text_token_prompt["prompt_token_ids"],
|
text_token_prompt["prompt_token_ids"],
|
||||||
prompt=input_text,
|
prompt=input_text,
|
||||||
)
|
)
|
||||||
@@ -125,19 +125,19 @@ class ServingScores(OpenAIServing):
|
|||||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||||
pooling_params = request.to_pooling_params("embed")
|
pooling_params = request.to_pooling_params("embed")
|
||||||
|
|
||||||
for i, engine_prompt in enumerate(engine_prompts):
|
for i, engine_input in enumerate(engine_inputs):
|
||||||
request_id_item = f"{request_id}-{i}"
|
request_id_item = f"{request_id}-{i}"
|
||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id_item,
|
request_id_item,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=pooling_params,
|
params=pooling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
generators.append(
|
generators.append(
|
||||||
self.engine_client.encode(
|
self.engine_client.encode(
|
||||||
engine_prompt,
|
engine_input,
|
||||||
pooling_params,
|
pooling_params,
|
||||||
request_id_item,
|
request_id_item,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -151,7 +151,7 @@ class ServingScores(OpenAIServing):
|
|||||||
# Non-streaming response
|
# Non-streaming response
|
||||||
final_res_batch: list[PoolingRequestOutput] = []
|
final_res_batch: list[PoolingRequestOutput] = []
|
||||||
|
|
||||||
embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_prompts)
|
embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_inputs)
|
||||||
|
|
||||||
async for i, res in result_generator:
|
async for i, res in result_generator:
|
||||||
embeddings[i] = res
|
embeddings[i] = res
|
||||||
@@ -183,7 +183,7 @@ class ServingScores(OpenAIServing):
|
|||||||
request: RerankRequest | ScoreRequest,
|
request: RerankRequest | ScoreRequest,
|
||||||
tokenizer: TokenizerLike,
|
tokenizer: TokenizerLike,
|
||||||
tokenization_kwargs: dict[str, Any],
|
tokenization_kwargs: dict[str, Any],
|
||||||
) -> tuple[str, TokensPrompt]:
|
) -> TokensPrompt:
|
||||||
"""Parse a single ScoreData into a text + optional multimodal
|
"""Parse a single ScoreData into a text + optional multimodal
|
||||||
TokensPrompt for late-interaction encoding.
|
TokensPrompt for late-interaction encoding.
|
||||||
|
|
||||||
@@ -197,21 +197,22 @@ class ServingScores(OpenAIServing):
|
|||||||
else:
|
else:
|
||||||
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
|
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
|
||||||
|
|
||||||
prompt_inputs = tokenizer(text, **tokenization_kwargs)
|
prompt_ids = tokenizer.encode(text, **tokenization_kwargs)
|
||||||
self._validate_input(request, prompt_inputs["input_ids"], text)
|
self._validate_input(request, prompt_ids, text)
|
||||||
|
|
||||||
engine_prompt = TokensPrompt(
|
tok_prompt = TokensPrompt(
|
||||||
prompt_token_ids=prompt_inputs["input_ids"],
|
prompt_token_ids=prompt_ids,
|
||||||
|
prompt=text,
|
||||||
)
|
)
|
||||||
|
|
||||||
if mm_data is not None:
|
if mm_data is not None:
|
||||||
engine_prompt["multi_modal_data"] = mm_data
|
tok_prompt["multi_modal_data"] = mm_data
|
||||||
if mm_uuids is not None:
|
if mm_uuids is not None:
|
||||||
engine_prompt["multi_modal_uuids"] = mm_uuids
|
tok_prompt["multi_modal_uuids"] = mm_uuids
|
||||||
if request.mm_processor_kwargs is not None:
|
if request.mm_processor_kwargs is not None:
|
||||||
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
|
tok_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
|
||||||
|
|
||||||
return text, engine_prompt
|
return tok_prompt
|
||||||
|
|
||||||
async def _late_interaction_score(
|
async def _late_interaction_score(
|
||||||
self,
|
self,
|
||||||
@@ -240,7 +241,7 @@ class ServingScores(OpenAIServing):
|
|||||||
executor=self._tokenizer_executor,
|
executor=self._tokenizer_executor,
|
||||||
)
|
)
|
||||||
|
|
||||||
preprocessed = await asyncio.gather(
|
tok_prompts = await asyncio.gather(
|
||||||
*(
|
*(
|
||||||
preprocess_async(
|
preprocess_async(
|
||||||
data=d,
|
data=d,
|
||||||
@@ -253,12 +254,8 @@ class ServingScores(OpenAIServing):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
query_prompts: list[TokensPrompt] = [
|
query_prompts = tok_prompts[: len(data_1)]
|
||||||
prompt for _, prompt in preprocessed[: len(data_1)]
|
doc_prompts = tok_prompts[len(data_1) :]
|
||||||
]
|
|
||||||
doc_prompts: list[TokensPrompt] = [
|
|
||||||
prompt for _, prompt in preprocessed[len(data_1) :]
|
|
||||||
]
|
|
||||||
|
|
||||||
default_pooling_params = request.to_pooling_params("token_embed")
|
default_pooling_params = request.to_pooling_params("token_embed")
|
||||||
|
|
||||||
@@ -268,7 +265,7 @@ class ServingScores(OpenAIServing):
|
|||||||
query_prompts
|
query_prompts
|
||||||
)
|
)
|
||||||
query_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
query_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||||
for i, engine_prompt in enumerate(query_prompts):
|
for i, tok_prompt in enumerate(query_prompts):
|
||||||
request_id_item = f"{request_id}-query-{i}"
|
request_id_item = f"{request_id}-query-{i}"
|
||||||
pooling_params = default_pooling_params.clone()
|
pooling_params = default_pooling_params.clone()
|
||||||
pooling_params.late_interaction_params = (
|
pooling_params.late_interaction_params = (
|
||||||
@@ -280,14 +277,14 @@ class ServingScores(OpenAIServing):
|
|||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id_item,
|
request_id_item,
|
||||||
engine_prompt,
|
tok_prompt,
|
||||||
params=pooling_params,
|
params=pooling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
query_generators.append(
|
query_generators.append(
|
||||||
self.engine_client.encode(
|
self.engine_client.encode(
|
||||||
engine_prompt,
|
tok_prompt,
|
||||||
pooling_params,
|
pooling_params,
|
||||||
request_id_item,
|
request_id_item,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -306,7 +303,7 @@ class ServingScores(OpenAIServing):
|
|||||||
|
|
||||||
# stage 2: encode docs and return scalar scores from workers.
|
# stage 2: encode docs and return scalar scores from workers.
|
||||||
doc_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
doc_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||||
for i, engine_prompt in enumerate(doc_prompts):
|
for i, tok_prompt in enumerate(doc_prompts):
|
||||||
request_id_item = f"{request_id}-doc-{i}"
|
request_id_item = f"{request_id}-doc-{i}"
|
||||||
query_idx = 0 if len(query_prompts) == 1 else i
|
query_idx = 0 if len(query_prompts) == 1 else i
|
||||||
pooling_params = default_pooling_params.clone()
|
pooling_params = default_pooling_params.clone()
|
||||||
@@ -316,14 +313,14 @@ class ServingScores(OpenAIServing):
|
|||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id_item,
|
request_id_item,
|
||||||
engine_prompt,
|
tok_prompt,
|
||||||
params=pooling_params,
|
params=pooling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
doc_generators.append(
|
doc_generators.append(
|
||||||
self.engine_client.encode(
|
self.engine_client.encode(
|
||||||
engine_prompt,
|
tok_prompt,
|
||||||
pooling_params,
|
pooling_params,
|
||||||
request_id_item,
|
request_id_item,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -404,28 +401,22 @@ class ServingScores(OpenAIServing):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
request_prompts: list[str] = []
|
|
||||||
engine_prompts: list[TokensPrompt] = []
|
|
||||||
for full_prompt, engine_prompt in preprocessed_prompts:
|
|
||||||
request_prompts.append(full_prompt)
|
|
||||||
engine_prompts.append(engine_prompt)
|
|
||||||
|
|
||||||
# Schedule the request and get the result generator.
|
# Schedule the request and get the result generator.
|
||||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||||
|
|
||||||
default_pooling_params = request.to_pooling_params("classify")
|
default_pooling_params = request.to_pooling_params("classify")
|
||||||
|
|
||||||
for i, engine_prompt in enumerate(engine_prompts):
|
for i, (full_prompt, tok_prompt) in enumerate(preprocessed_prompts):
|
||||||
request_id_item = f"{request_id}-{i}"
|
request_id_item = f"{request_id}-{i}"
|
||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id_item,
|
request_id_item,
|
||||||
request_prompts[i],
|
full_prompt,
|
||||||
params=default_pooling_params,
|
params=default_pooling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
if token_type_ids := engine_prompt.pop("token_type_ids", None):
|
if token_type_ids := tok_prompt.pop("token_type_ids", None):
|
||||||
pooling_params = default_pooling_params.clone()
|
pooling_params = default_pooling_params.clone()
|
||||||
compressed = compress_token_type_ids(token_type_ids)
|
compressed = compress_token_type_ids(token_type_ids)
|
||||||
pooling_params.extra_kwargs = {"compressed_token_type_ids": compressed}
|
pooling_params.extra_kwargs = {"compressed_token_type_ids": compressed}
|
||||||
@@ -433,7 +424,7 @@ class ServingScores(OpenAIServing):
|
|||||||
pooling_params = default_pooling_params
|
pooling_params = default_pooling_params
|
||||||
|
|
||||||
generator = self.engine_client.encode(
|
generator = self.engine_client.encode(
|
||||||
engine_prompt,
|
tok_prompt,
|
||||||
pooling_params,
|
pooling_params,
|
||||||
request_id_item,
|
request_id_item,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
@@ -447,7 +438,7 @@ class ServingScores(OpenAIServing):
|
|||||||
|
|
||||||
# Non-streaming response
|
# Non-streaming response
|
||||||
final_res_batch: list[PoolingRequestOutput | None] = [None] * len(
|
final_res_batch: list[PoolingRequestOutput | None] = [None] * len(
|
||||||
engine_prompts
|
preprocessed_prompts
|
||||||
)
|
)
|
||||||
|
|
||||||
async for i, res in result_generator:
|
async for i, res in result_generator:
|
||||||
@@ -464,7 +455,7 @@ class ServingScores(OpenAIServing):
|
|||||||
data_2: ScoreData,
|
data_2: ScoreData,
|
||||||
) -> tuple[str, TokensPrompt]:
|
) -> tuple[str, TokensPrompt]:
|
||||||
model_config = self.model_config
|
model_config = self.model_config
|
||||||
full_prompt, engine_prompt = get_score_prompt(
|
full_prompt, engine_input = get_score_prompt(
|
||||||
model_config=model_config,
|
model_config=model_config,
|
||||||
data_1=data_1,
|
data_1=data_1,
|
||||||
data_2=data_2,
|
data_2=data_2,
|
||||||
@@ -472,11 +463,11 @@ class ServingScores(OpenAIServing):
|
|||||||
tokenization_kwargs=tokenization_kwargs,
|
tokenization_kwargs=tokenization_kwargs,
|
||||||
score_template=self.score_template,
|
score_template=self.score_template,
|
||||||
)
|
)
|
||||||
self._validate_input(request, engine_prompt["prompt_token_ids"], full_prompt)
|
self._validate_input(request, engine_input["prompt_token_ids"], full_prompt)
|
||||||
if request.mm_processor_kwargs is not None:
|
if request.mm_processor_kwargs is not None:
|
||||||
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
|
engine_input["mm_processor_kwargs"] = request.mm_processor_kwargs
|
||||||
|
|
||||||
return full_prompt, engine_prompt
|
return full_prompt, engine_input
|
||||||
|
|
||||||
async def _run_scoring(
|
async def _run_scoring(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -20,10 +20,14 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
MultiModalItemTracker,
|
MultiModalItemTracker,
|
||||||
_parse_chat_message_content_parts,
|
_parse_chat_message_content_parts,
|
||||||
)
|
)
|
||||||
from vllm.inputs import TokensPrompt
|
from vllm.inputs import (
|
||||||
from vllm.inputs.data import PromptType, TextPrompt
|
MultiModalDataDict,
|
||||||
|
MultiModalUUIDDict,
|
||||||
|
PromptType,
|
||||||
|
TextPrompt,
|
||||||
|
TokensPrompt,
|
||||||
|
)
|
||||||
from vllm.model_executor.models.interfaces import supports_score_template
|
from vllm.model_executor.models.interfaces import supports_score_template
|
||||||
from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict
|
|
||||||
from vllm.outputs import PoolingRequestOutput
|
from vllm.outputs import PoolingRequestOutput
|
||||||
from vllm.renderers.hf import safe_apply_chat_template
|
from vllm.renderers.hf import safe_apply_chat_template
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ from vllm.entrypoints.pooling.score.protocol import (
|
|||||||
ScoreRequest,
|
ScoreRequest,
|
||||||
ScoreResponse,
|
ScoreResponse,
|
||||||
)
|
)
|
||||||
from vllm.inputs import ProcessorInputs
|
from vllm.inputs import EngineInput
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
|
|
||||||
PoolingCompletionLikeRequest: TypeAlias = (
|
PoolingCompletionLikeRequest: TypeAlias = (
|
||||||
@@ -74,7 +74,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
|||||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||||
lora_request: LoRARequest | None = None
|
lora_request: LoRARequest | None = None
|
||||||
|
|
||||||
engine_prompts: list[ProcessorInputs] | None = None
|
engine_inputs: list[EngineInput] | None = None
|
||||||
prompt_request_ids: list[str] | None = None
|
prompt_request_ids: list[str] | None = None
|
||||||
intermediates: Any | None = None
|
intermediates: Any | None = None
|
||||||
|
|
||||||
|
|||||||
@@ -33,19 +33,20 @@ class MultiModalFeatures(BaseModel):
|
|||||||
"""Lightweight multimodal metadata produced by the render step.
|
"""Lightweight multimodal metadata produced by the render step.
|
||||||
|
|
||||||
Carries hashes (for cache lookup / identification) and placeholder
|
Carries hashes (for cache lookup / identification) and placeholder
|
||||||
positions so the downstream ``/generate`` service knows *where* in
|
positions so the downstream `/generate` service knows *where* in
|
||||||
the token sequence each multimodal item lives.
|
the token sequence each multimodal item lives.
|
||||||
|
|
||||||
.. note:: Phase 1 — metadata only.
|
Note:
|
||||||
Phase 2 should add ``mm_kwargs`` (processed tensor data) using a
|
Phase 1 — metadata only.
|
||||||
binary transport so the ``/generate`` side can skip re-processing.
|
Phase 2 should add `mm_kwargs` (processed tensor data) using a
|
||||||
The ``/generate`` endpoint must also be updated to inject these
|
binary transport so the ``/generate` side can skip re-processing.
|
||||||
features into ``ProcessorInputs`` before passing to
|
The `/generate` endpoint must also be updated to inject these
|
||||||
``InputProcessor.process_inputs``.
|
features into `EngineInput` before passing to
|
||||||
|
`InputProcessor.process_inputs`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
mm_hashes: dict[str, list[str]]
|
mm_hashes: dict[str, list[str]]
|
||||||
"""Per-modality item hashes, e.g. ``{"image": ["abc", "def"]}``."""
|
"""Per-modality item hashes, e.g. `{"image": ["abc", "def"]}`."""
|
||||||
|
|
||||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]]
|
mm_placeholders: dict[str, list[PlaceholderRangeInfo]]
|
||||||
"""Per-modality placeholder ranges in the token sequence."""
|
"""Per-modality placeholder ranges in the token sequence."""
|
||||||
|
|||||||
@@ -99,13 +99,11 @@ class ServingTokens(OpenAIServing):
|
|||||||
if raw_request:
|
if raw_request:
|
||||||
raw_request.state.request_metadata = request_metadata
|
raw_request.state.request_metadata = request_metadata
|
||||||
|
|
||||||
engine_prompts = await self.openai_serving_render.preprocess_completion(
|
(engine_input,) = await self.openai_serving_render.preprocess_completion(
|
||||||
request,
|
request,
|
||||||
prompt_input=request.token_ids,
|
prompt_input=request.token_ids,
|
||||||
prompt_embeds=None,
|
prompt_embeds=None,
|
||||||
)
|
)
|
||||||
assert len(engine_prompts) == 1
|
|
||||||
engine_prompt = engine_prompts[0]
|
|
||||||
|
|
||||||
# Schedule the request and get the result generator.
|
# Schedule the request and get the result generator.
|
||||||
result_generator: AsyncGenerator[RequestOutput, None] | None = None
|
result_generator: AsyncGenerator[RequestOutput, None] | None = None
|
||||||
@@ -115,7 +113,7 @@ class ServingTokens(OpenAIServing):
|
|||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id,
|
request_id,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=sampling_params,
|
params=sampling_params,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
@@ -127,7 +125,7 @@ class ServingTokens(OpenAIServing):
|
|||||||
)
|
)
|
||||||
|
|
||||||
result_generator = self.engine_client.generate(
|
result_generator = self.engine_client.generate(
|
||||||
engine_prompt,
|
engine_input,
|
||||||
sampling_params,
|
sampling_params,
|
||||||
request_id,
|
request_id,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
|
|||||||
@@ -34,9 +34,15 @@ from vllm.entrypoints.utils import (
|
|||||||
create_error_response,
|
create_error_response,
|
||||||
get_max_tokens,
|
get_max_tokens,
|
||||||
)
|
)
|
||||||
from vllm.inputs.data import ProcessorInputs, PromptType, SingletonPrompt, TokensPrompt
|
from vllm.inputs import (
|
||||||
|
EngineInput,
|
||||||
|
MultiModalHashes,
|
||||||
|
MultiModalPlaceholders,
|
||||||
|
PromptType,
|
||||||
|
SingletonPrompt,
|
||||||
|
tokens_input,
|
||||||
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.multimodal.inputs import MultiModalHashes, MultiModalPlaceholderDict
|
|
||||||
from vllm.parser import ParserManager
|
from vllm.parser import ParserManager
|
||||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||||
from vllm.renderers.inputs.preprocess import (
|
from vllm.renderers.inputs.preprocess import (
|
||||||
@@ -127,22 +133,22 @@ class OpenAIServingRender:
|
|||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
_, engine_prompts = result
|
_, engine_inputs = result
|
||||||
|
|
||||||
if len(engine_prompts) != 1:
|
if len(engine_inputs) != 1:
|
||||||
return self.create_error_response(
|
return self.create_error_response(
|
||||||
f"Expected exactly 1 engine prompt, got {len(engine_prompts)}"
|
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
|
||||||
)
|
)
|
||||||
|
|
||||||
engine_prompt = engine_prompts[0]
|
engine_input = engine_inputs[0]
|
||||||
|
|
||||||
prompt_components = extract_prompt_components(self.model_config, engine_prompt)
|
prompt_components = extract_prompt_components(self.model_config, engine_input)
|
||||||
token_ids = prompt_components.token_ids
|
token_ids = prompt_components.token_ids
|
||||||
if not token_ids:
|
if not token_ids:
|
||||||
return self.create_error_response("No token_ids rendered")
|
return self.create_error_response("No token_ids rendered")
|
||||||
token_ids = list(token_ids)
|
token_ids = list(token_ids)
|
||||||
|
|
||||||
input_length = extract_prompt_len(self.model_config, engine_prompt)
|
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||||
max_tokens = get_max_tokens(
|
max_tokens = get_max_tokens(
|
||||||
self.model_config.max_model_len,
|
self.model_config.max_model_len,
|
||||||
request.max_completion_tokens
|
request.max_completion_tokens
|
||||||
@@ -159,7 +165,7 @@ class OpenAIServingRender:
|
|||||||
return GenerateRequest(
|
return GenerateRequest(
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
token_ids=token_ids,
|
token_ids=token_ids,
|
||||||
features=self._extract_mm_features(engine_prompt),
|
features=self._extract_mm_features(engine_input),
|
||||||
sampling_params=params,
|
sampling_params=params,
|
||||||
model=request.model,
|
model=request.model,
|
||||||
stream=bool(request.stream),
|
stream=bool(request.stream),
|
||||||
@@ -171,7 +177,7 @@ class OpenAIServingRender:
|
|||||||
async def render_chat(
|
async def render_chat(
|
||||||
self,
|
self,
|
||||||
request: ChatCompletionRequest,
|
request: ChatCompletionRequest,
|
||||||
) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
|
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
|
||||||
"""Core preprocessing logic for chat requests (no model/engine check).
|
"""Core preprocessing logic for chat requests (no model/engine check).
|
||||||
|
|
||||||
Called directly by render_chat_request and delegated to by
|
Called directly by render_chat_request and delegated to by
|
||||||
@@ -184,7 +190,6 @@ class OpenAIServingRender:
|
|||||||
if is_mistral_tokenizer(tokenizer):
|
if is_mistral_tokenizer(tokenizer):
|
||||||
# because of issues with pydantic we need to potentially
|
# because of issues with pydantic we need to potentially
|
||||||
# re-serialize the tool_calls field of the request
|
# re-serialize the tool_calls field of the request
|
||||||
# for more info: see comment in `maybe_serialize_tool_calls`
|
|
||||||
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
|
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
|
||||||
_mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
|
_mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
|
||||||
_mt.validate_request_params(request)
|
_mt.validate_request_params(request)
|
||||||
@@ -232,7 +237,7 @@ class OpenAIServingRender:
|
|||||||
if error_check_ret is not None:
|
if error_check_ret is not None:
|
||||||
return error_check_ret
|
return error_check_ret
|
||||||
|
|
||||||
conversation, engine_prompts = await self.preprocess_chat(
|
conversation, engine_inputs = await self.preprocess_chat(
|
||||||
request,
|
request,
|
||||||
request.messages,
|
request.messages,
|
||||||
default_template=self.chat_template,
|
default_template=self.chat_template,
|
||||||
@@ -244,11 +249,11 @@ class OpenAIServingRender:
|
|||||||
else:
|
else:
|
||||||
# For GPT-OSS.
|
# For GPT-OSS.
|
||||||
should_include_tools = tool_dicts is not None
|
should_include_tools = tool_dicts is not None
|
||||||
conversation, engine_prompts = self._make_request_with_harmony(
|
conversation, engine_inputs = self._make_request_with_harmony(
|
||||||
request, should_include_tools
|
request, should_include_tools
|
||||||
)
|
)
|
||||||
|
|
||||||
return conversation, engine_prompts
|
return conversation, engine_inputs
|
||||||
|
|
||||||
async def render_completion_request(
|
async def render_completion_request(
|
||||||
self,
|
self,
|
||||||
@@ -266,16 +271,16 @@ class OpenAIServingRender:
|
|||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
return result
|
return result
|
||||||
generate_requests: list[GenerateRequest] = []
|
generate_requests: list[GenerateRequest] = []
|
||||||
for engine_prompt in result:
|
for engine_input in result:
|
||||||
prompt_components = extract_prompt_components(
|
prompt_components = extract_prompt_components(
|
||||||
self.model_config, engine_prompt
|
self.model_config, engine_input
|
||||||
)
|
)
|
||||||
token_ids = prompt_components.token_ids
|
token_ids = prompt_components.token_ids
|
||||||
if not token_ids:
|
if not token_ids:
|
||||||
return self.create_error_response("No token_ids rendered")
|
return self.create_error_response("No token_ids rendered")
|
||||||
token_ids = list(token_ids)
|
token_ids = list(token_ids)
|
||||||
|
|
||||||
input_length = extract_prompt_len(self.model_config, engine_prompt)
|
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||||
max_tokens = get_max_tokens(
|
max_tokens = get_max_tokens(
|
||||||
self.model_config.max_model_len,
|
self.model_config.max_model_len,
|
||||||
request.max_tokens,
|
request.max_tokens,
|
||||||
@@ -293,7 +298,7 @@ class OpenAIServingRender:
|
|||||||
GenerateRequest(
|
GenerateRequest(
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
token_ids=token_ids,
|
token_ids=token_ids,
|
||||||
features=self._extract_mm_features(engine_prompt),
|
features=self._extract_mm_features(engine_input),
|
||||||
sampling_params=params,
|
sampling_params=params,
|
||||||
model=request.model,
|
model=request.model,
|
||||||
stream=bool(request.stream),
|
stream=bool(request.stream),
|
||||||
@@ -308,7 +313,7 @@ class OpenAIServingRender:
|
|||||||
async def render_completion(
|
async def render_completion(
|
||||||
self,
|
self,
|
||||||
request: CompletionRequest,
|
request: CompletionRequest,
|
||||||
) -> list[ProcessorInputs] | ErrorResponse:
|
) -> list[EngineInput] | ErrorResponse:
|
||||||
"""Core preprocessing logic for completion requests (no model/engine check).
|
"""Core preprocessing logic for completion requests (no model/engine check).
|
||||||
|
|
||||||
Called directly by render_completion_request and delegated to by
|
Called directly by render_completion_request and delegated to by
|
||||||
@@ -326,28 +331,28 @@ class OpenAIServingRender:
|
|||||||
"prompt_logprobs is not compatible with prompt embeds."
|
"prompt_logprobs is not compatible with prompt embeds."
|
||||||
)
|
)
|
||||||
|
|
||||||
engine_prompts = await self.preprocess_completion(
|
engine_inputs = await self.preprocess_completion(
|
||||||
request,
|
request,
|
||||||
prompt_input=request.prompt,
|
prompt_input=request.prompt,
|
||||||
prompt_embeds=request.prompt_embeds,
|
prompt_embeds=request.prompt_embeds,
|
||||||
)
|
)
|
||||||
|
|
||||||
return engine_prompts
|
return engine_inputs
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_mm_features(
|
def _extract_mm_features(
|
||||||
engine_prompt: ProcessorInputs,
|
engine_input: EngineInput,
|
||||||
) -> MultiModalFeatures | None:
|
) -> MultiModalFeatures | None:
|
||||||
"""Extract multimodal metadata from a rendered engine prompt.
|
"""Extract multimodal metadata from a rendered engine prompt.
|
||||||
|
|
||||||
Returns ``None`` for text-only prompts.
|
Returns ``None`` for text-only prompts.
|
||||||
"""
|
"""
|
||||||
if engine_prompt.get("type") != "multimodal":
|
if engine_input.get("type") != "multimodal":
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# At this point engine_prompt is a MultiModalInputs TypedDict.
|
# At this point engine_input is a MultiModalInputs TypedDict.
|
||||||
mm_hashes: MultiModalHashes = engine_prompt["mm_hashes"] # type: ignore[typeddict-item]
|
mm_hashes: MultiModalHashes = engine_input["mm_hashes"] # type: ignore[typeddict-item]
|
||||||
raw_placeholders: MultiModalPlaceholderDict = engine_prompt["mm_placeholders"] # type: ignore[typeddict-item]
|
raw_placeholders: MultiModalPlaceholders = engine_input["mm_placeholders"] # type: ignore[typeddict-item]
|
||||||
|
|
||||||
mm_placeholders = {
|
mm_placeholders = {
|
||||||
modality: [
|
modality: [
|
||||||
@@ -401,13 +406,9 @@ class OpenAIServingRender:
|
|||||||
|
|
||||||
# Render prompt token ids.
|
# Render prompt token ids.
|
||||||
prompt_token_ids = render_for_completion(messages)
|
prompt_token_ids = render_for_completion(messages)
|
||||||
engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
|
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
|
||||||
|
|
||||||
# Add cache_salt if provided in the request
|
return messages, [engine_input]
|
||||||
if request.cache_salt is not None:
|
|
||||||
engine_prompt["cache_salt"] = request.cache_salt
|
|
||||||
|
|
||||||
return messages, [engine_prompt]
|
|
||||||
|
|
||||||
def create_error_response(
|
def create_error_response(
|
||||||
self,
|
self,
|
||||||
@@ -450,7 +451,7 @@ class OpenAIServingRender:
|
|||||||
request: Any,
|
request: Any,
|
||||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||||
prompt_embeds: bytes | list[bytes] | None,
|
prompt_embeds: bytes | list[bytes] | None,
|
||||||
) -> list[ProcessorInputs]:
|
) -> list[EngineInput]:
|
||||||
"""Copied from OpenAIServing._preprocess_completion."""
|
"""Copied from OpenAIServing._preprocess_completion."""
|
||||||
prompts = list[SingletonPrompt | bytes]()
|
prompts = list[SingletonPrompt | bytes]()
|
||||||
if prompt_embeds is not None: # embeds take higher priority
|
if prompt_embeds is not None: # embeds take higher priority
|
||||||
@@ -463,7 +464,7 @@ class OpenAIServingRender:
|
|||||||
self,
|
self,
|
||||||
request: Any,
|
request: Any,
|
||||||
prompts: Sequence[PromptType | bytes],
|
prompts: Sequence[PromptType | bytes],
|
||||||
) -> list[ProcessorInputs]:
|
) -> list[EngineInput]:
|
||||||
"""Copied from OpenAIServing._preprocess_cmpl."""
|
"""Copied from OpenAIServing._preprocess_cmpl."""
|
||||||
renderer = self.renderer
|
renderer = self.renderer
|
||||||
model_config = self.model_config
|
model_config = self.model_config
|
||||||
@@ -497,7 +498,7 @@ class OpenAIServingRender:
|
|||||||
default_template_kwargs: dict[str, Any] | None,
|
default_template_kwargs: dict[str, Any] | None,
|
||||||
tool_dicts: list[dict[str, Any]] | None = None,
|
tool_dicts: list[dict[str, Any]] | None = None,
|
||||||
tool_parser: type[ToolParser] | None = None,
|
tool_parser: type[ToolParser] | None = None,
|
||||||
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
|
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||||
"""Copied from OpenAIServing._preprocess_chat."""
|
"""Copied from OpenAIServing._preprocess_chat."""
|
||||||
renderer = self.renderer
|
renderer = self.renderer
|
||||||
mm_config = self.model_config.multimodal_config
|
mm_config = self.model_config.multimodal_config
|
||||||
@@ -519,7 +520,7 @@ class OpenAIServingRender:
|
|||||||
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||||
)
|
)
|
||||||
|
|
||||||
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
|
(conversation,), (engine_input,) = await renderer.render_chat_async(
|
||||||
[messages],
|
[messages],
|
||||||
chat_params,
|
chat_params,
|
||||||
tok_params,
|
tok_params,
|
||||||
@@ -546,4 +547,4 @@ class OpenAIServingRender:
|
|||||||
tokenizer = renderer.get_tokenizer()
|
tokenizer = renderer.get_tokenizer()
|
||||||
request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore[arg-type]
|
request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore[arg-type]
|
||||||
|
|
||||||
return conversation, [engine_prompt]
|
return conversation, [engine_input]
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
|
|||||||
TokenizeResponse,
|
TokenizeResponse,
|
||||||
TokenizerInfoResponse,
|
TokenizerInfoResponse,
|
||||||
)
|
)
|
||||||
from vllm.inputs import TokensPrompt, token_inputs
|
from vllm.inputs import TokensPrompt, tokens_input
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
|
||||||
@@ -79,7 +79,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
|||||||
if error_check_ret is not None:
|
if error_check_ret is not None:
|
||||||
return error_check_ret
|
return error_check_ret
|
||||||
|
|
||||||
_, engine_prompts = await self.openai_serving_render.preprocess_chat(
|
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||||
request,
|
request,
|
||||||
request.messages,
|
request.messages,
|
||||||
default_template=self.chat_template,
|
default_template=self.chat_template,
|
||||||
@@ -88,22 +88,22 @@ class OpenAIServingTokenization(OpenAIServing):
|
|||||||
tool_dicts=tool_dicts,
|
tool_dicts=tool_dicts,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
engine_prompts = await self.openai_serving_render.preprocess_completion(
|
engine_inputs = await self.openai_serving_render.preprocess_completion(
|
||||||
request,
|
request,
|
||||||
prompt_input=request.prompt,
|
prompt_input=request.prompt,
|
||||||
prompt_embeds=None,
|
prompt_embeds=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
input_ids: list[int] = []
|
input_ids: list[int] = []
|
||||||
for engine_prompt in engine_prompts:
|
for engine_input in engine_inputs:
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id,
|
request_id,
|
||||||
engine_prompt,
|
engine_input,
|
||||||
params=None,
|
params=None,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
prompt_components = self._extract_prompt_components(engine_prompt)
|
prompt_components = self._extract_prompt_components(engine_input)
|
||||||
if prompt_components.token_ids is not None:
|
if prompt_components.token_ids is not None:
|
||||||
input_ids.extend(prompt_components.token_ids)
|
input_ids.extend(prompt_components.token_ids)
|
||||||
|
|
||||||
@@ -134,16 +134,16 @@ class OpenAIServingTokenization(OpenAIServing):
|
|||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id,
|
request_id,
|
||||||
token_inputs(request.tokens),
|
tokens_input(request.tokens),
|
||||||
params=None,
|
params=None,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
engine_prompt = await self.renderer.tokenize_prompt_async(
|
tok_prompt = await self.renderer.tokenize_prompt_async(
|
||||||
TokensPrompt(prompt_token_ids=request.tokens),
|
TokensPrompt(prompt_token_ids=request.tokens),
|
||||||
request.build_tok_params(self.model_config),
|
request.build_tok_params(self.model_config),
|
||||||
)
|
)
|
||||||
prompt_text = engine_prompt["prompt"] # type: ignore[typeddict-item]
|
prompt_text = tok_prompt["prompt"] # type: ignore[typeddict-item]
|
||||||
|
|
||||||
return DetokenizeResponse(prompt=prompt_text)
|
return DetokenizeResponse(prompt=prompt_text)
|
||||||
|
|
||||||
|
|||||||
+43
-17
@@ -1,38 +1,64 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from .data import (
|
from .engine import (
|
||||||
|
DecoderOnlyEngineInput,
|
||||||
|
EmbedsInput,
|
||||||
|
EncoderDecoderInput,
|
||||||
|
EngineInput,
|
||||||
|
MultiModalEncDecInput,
|
||||||
|
MultiModalHashes,
|
||||||
|
MultiModalInput,
|
||||||
|
MultiModalPlaceholders,
|
||||||
|
SingletonInput,
|
||||||
|
TokensInput,
|
||||||
|
build_enc_dec_input,
|
||||||
|
embeds_input,
|
||||||
|
mm_enc_dec_input,
|
||||||
|
mm_input,
|
||||||
|
split_enc_dec_input,
|
||||||
|
tokens_input,
|
||||||
|
)
|
||||||
|
from .llm import (
|
||||||
DataPrompt,
|
DataPrompt,
|
||||||
DecoderOnlyInputs,
|
|
||||||
EmbedsInputs,
|
|
||||||
EmbedsPrompt,
|
EmbedsPrompt,
|
||||||
EncoderDecoderInputs,
|
|
||||||
ExplicitEncoderDecoderPrompt,
|
ExplicitEncoderDecoderPrompt,
|
||||||
ProcessorInputs,
|
ModalityData,
|
||||||
|
MultiModalDataBuiltins,
|
||||||
|
MultiModalDataDict,
|
||||||
|
MultiModalUUIDDict,
|
||||||
PromptType,
|
PromptType,
|
||||||
SingletonInputs,
|
|
||||||
SingletonPrompt,
|
SingletonPrompt,
|
||||||
TextPrompt,
|
TextPrompt,
|
||||||
TokenInputs,
|
|
||||||
TokensPrompt,
|
TokensPrompt,
|
||||||
embeds_inputs,
|
|
||||||
token_inputs,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
"ModalityData",
|
||||||
|
"MultiModalDataBuiltins",
|
||||||
|
"MultiModalDataDict",
|
||||||
|
"MultiModalUUIDDict",
|
||||||
"DataPrompt",
|
"DataPrompt",
|
||||||
"TextPrompt",
|
"TextPrompt",
|
||||||
"TokensPrompt",
|
"TokensPrompt",
|
||||||
"PromptType",
|
"PromptType",
|
||||||
"SingletonPrompt",
|
"SingletonPrompt",
|
||||||
"ExplicitEncoderDecoderPrompt",
|
"ExplicitEncoderDecoderPrompt",
|
||||||
"TokenInputs",
|
|
||||||
"EmbedsInputs",
|
|
||||||
"EmbedsPrompt",
|
"EmbedsPrompt",
|
||||||
"token_inputs",
|
"MultiModalHashes",
|
||||||
"embeds_inputs",
|
"MultiModalPlaceholders",
|
||||||
"DecoderOnlyInputs",
|
"TokensInput",
|
||||||
"EncoderDecoderInputs",
|
"EmbedsInput",
|
||||||
"ProcessorInputs",
|
"MultiModalInput",
|
||||||
"SingletonInputs",
|
"MultiModalEncDecInput",
|
||||||
|
"tokens_input",
|
||||||
|
"embeds_input",
|
||||||
|
"mm_input",
|
||||||
|
"mm_enc_dec_input",
|
||||||
|
"build_enc_dec_input",
|
||||||
|
"split_enc_dec_input",
|
||||||
|
"DecoderOnlyEngineInput",
|
||||||
|
"EncoderDecoderInput",
|
||||||
|
"SingletonInput",
|
||||||
|
"EngineInput",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,413 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
from typing import TYPE_CHECKING, Any, Literal, TypeAlias
|
|
||||||
|
|
||||||
import torch
|
|
||||||
from typing_extensions import NotRequired, TypedDict, assert_never
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from vllm.multimodal.inputs import (
|
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalEncDecInputs,
|
|
||||||
MultiModalInputs,
|
|
||||||
MultiModalUUIDDict,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
MultiModalDataDict = object
|
|
||||||
MultiModalEncDecInputs = object
|
|
||||||
MultiModalInputs = object
|
|
||||||
MultiModalUUIDDict = object
|
|
||||||
|
|
||||||
|
|
||||||
# Inputs to LLM API
|
|
||||||
class _PromptOptions(TypedDict):
|
|
||||||
"""
|
|
||||||
Additional options available to all
|
|
||||||
[`SingletonPrompt`][vllm.inputs.data.SingletonPrompt].
|
|
||||||
"""
|
|
||||||
|
|
||||||
multi_modal_data: NotRequired[MultiModalDataDict | None]
|
|
||||||
"""
|
|
||||||
Optional multi-modal data to pass to the model,
|
|
||||||
if the model supports it.
|
|
||||||
"""
|
|
||||||
|
|
||||||
mm_processor_kwargs: NotRequired[dict[str, Any] | None]
|
|
||||||
"""
|
|
||||||
Optional multi-modal processor kwargs to be forwarded to the
|
|
||||||
multimodal input mapper & processor. Note that if multiple modalities
|
|
||||||
have registered mappers etc for the model being considered, we attempt
|
|
||||||
to pass the mm_processor_kwargs to each of them.
|
|
||||||
"""
|
|
||||||
|
|
||||||
multi_modal_uuids: NotRequired[MultiModalUUIDDict]
|
|
||||||
"""
|
|
||||||
Optional user-specified UUIDs for multimodal items, mapped by modality.
|
|
||||||
Lists must match the number of items per modality and may contain `None`.
|
|
||||||
For `None` entries, the hasher will compute IDs automatically; non-None
|
|
||||||
entries override the default hashes for caching, and MUST be unique per
|
|
||||||
multimodal item.
|
|
||||||
"""
|
|
||||||
|
|
||||||
cache_salt: NotRequired[str]
|
|
||||||
"""
|
|
||||||
Optional cache salt to be used for prefix caching.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class TextPrompt(_PromptOptions):
|
|
||||||
"""Schema for a text prompt."""
|
|
||||||
|
|
||||||
prompt: str
|
|
||||||
"""The input text to be tokenized before passing to the model."""
|
|
||||||
|
|
||||||
|
|
||||||
class TokensPrompt(_PromptOptions):
|
|
||||||
"""Schema for a tokenized prompt."""
|
|
||||||
|
|
||||||
prompt_token_ids: list[int]
|
|
||||||
"""A list of token IDs to pass to the model."""
|
|
||||||
|
|
||||||
prompt: NotRequired[str]
|
|
||||||
"""The prompt text corresponding to the token IDs, if available."""
|
|
||||||
|
|
||||||
token_type_ids: NotRequired[list[int]]
|
|
||||||
"""A list of token type IDs to pass to the cross encoder model."""
|
|
||||||
|
|
||||||
|
|
||||||
class EmbedsPrompt(_PromptOptions):
|
|
||||||
"""Schema for a prompt provided via token embeddings."""
|
|
||||||
|
|
||||||
prompt_embeds: torch.Tensor
|
|
||||||
"""The embeddings of the prompt."""
|
|
||||||
|
|
||||||
prompt: NotRequired[str]
|
|
||||||
"""The prompt text corresponding to the token embeddings, if available."""
|
|
||||||
|
|
||||||
|
|
||||||
DecoderOnlyPrompt: TypeAlias = (
|
|
||||||
str | TextPrompt | list[int] | TokensPrompt | EmbedsPrompt
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
Schema of a prompt for a decoder-only model:
|
|
||||||
|
|
||||||
- A text prompt (string or [`TextPrompt`][vllm.inputs.data.TextPrompt])
|
|
||||||
- A tokenized prompt (list of token IDs, or
|
|
||||||
[`TokensPrompt`][vllm.inputs.data.TokensPrompt])
|
|
||||||
- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])
|
|
||||||
|
|
||||||
For encoder-decoder models, passing a singleton prompt is shorthand for passing
|
|
||||||
`ExplicitEncoderDecoderPrompt(encoder_prompt=prompt, decoder_prompt=None)`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
EncoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt
|
|
||||||
"""
|
|
||||||
Schema of a prompt for the encoder part of a encoder-decoder model:
|
|
||||||
|
|
||||||
- A text prompt (string or [`TextPrompt`][vllm.inputs.data.TextPrompt])
|
|
||||||
- A tokenized prompt (list of token IDs, or
|
|
||||||
[`TokensPrompt`][vllm.inputs.data.TokensPrompt])
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
DecoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt
|
|
||||||
"""
|
|
||||||
Schema of a prompt for the decoder part of an encoder-decoder model:
|
|
||||||
|
|
||||||
- A text prompt (string or [`TextPrompt`][vllm.inputs.data.TextPrompt])
|
|
||||||
- A tokenized prompt (list of token IDs, or
|
|
||||||
[`TokensPrompt`][vllm.inputs.data.TokensPrompt])
|
|
||||||
|
|
||||||
Note:
|
|
||||||
Multi-modal inputs are not supported for decoder prompts.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class ExplicitEncoderDecoderPrompt(TypedDict):
|
|
||||||
"""
|
|
||||||
Schema for a pair of encoder and decoder singleton prompts.
|
|
||||||
|
|
||||||
Note:
|
|
||||||
This schema is not valid for decoder-only models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
encoder_prompt: EncoderPrompt
|
|
||||||
"""The prompt for the encoder part of the model."""
|
|
||||||
|
|
||||||
decoder_prompt: DecoderPrompt | None
|
|
||||||
"""
|
|
||||||
The prompt for the decoder part of the model.
|
|
||||||
|
|
||||||
Passing `None` will cause the prompt to be inferred automatically.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
EncoderDecoderPrompt: TypeAlias = EncoderPrompt | ExplicitEncoderDecoderPrompt
|
|
||||||
"""
|
|
||||||
Schema for a prompt for an encoder-decoder model.
|
|
||||||
|
|
||||||
You can pass a singleton encoder prompt, in which case the decoder prompt is
|
|
||||||
considered to be `None` (i.e., infer automatically).
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
SingletonPrompt: TypeAlias = DecoderOnlyPrompt | EncoderPrompt | DecoderPrompt
|
|
||||||
"""
|
|
||||||
Schema for a single prompt. This is as opposed to a data structure
|
|
||||||
which encapsulates multiple prompts, such as
|
|
||||||
[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt].
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
PromptType: TypeAlias = DecoderOnlyPrompt | EncoderDecoderPrompt
|
|
||||||
"""
|
|
||||||
Schema for any prompt, regardless of model type.
|
|
||||||
|
|
||||||
This is the input format accepted by most [`LLM`][vllm.entrypoints.llm.LLM] APIs.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class DataPrompt(_PromptOptions):
|
|
||||||
"""
|
|
||||||
Represents generic inputs that are converted to
|
|
||||||
[`PromptType`][vllm.inputs.data.PromptType] by IO processor plugins.
|
|
||||||
"""
|
|
||||||
|
|
||||||
data: Any
|
|
||||||
"""The input data."""
|
|
||||||
|
|
||||||
data_format: str
|
|
||||||
"""The input data format."""
|
|
||||||
|
|
||||||
|
|
||||||
# Outputs of processor
|
|
||||||
class _InputOptions(TypedDict):
|
|
||||||
"""
|
|
||||||
Additional options available to all input types.
|
|
||||||
"""
|
|
||||||
|
|
||||||
arrival_time: NotRequired[float]
|
|
||||||
"""The time when the input was received (before rendering)."""
|
|
||||||
|
|
||||||
cache_salt: NotRequired[str]
|
|
||||||
"""Optional cache salt to be used for prefix caching."""
|
|
||||||
|
|
||||||
|
|
||||||
class TokenInputs(_InputOptions):
|
|
||||||
"""Represents token-based inputs."""
|
|
||||||
|
|
||||||
type: Literal["token"]
|
|
||||||
"""The type of inputs."""
|
|
||||||
|
|
||||||
prompt_token_ids: list[int]
|
|
||||||
"""The token IDs of the prompt."""
|
|
||||||
|
|
||||||
prompt: NotRequired[str]
|
|
||||||
"""The prompt text corresponding to the token IDs, if available."""
|
|
||||||
|
|
||||||
|
|
||||||
def token_inputs(
|
|
||||||
prompt_token_ids: list[int],
|
|
||||||
*,
|
|
||||||
prompt: str | None = None,
|
|
||||||
cache_salt: str | None = None,
|
|
||||||
) -> TokenInputs:
|
|
||||||
"""Construct [`TokenInputs`][vllm.inputs.data.TokenInputs] from optional
|
|
||||||
values."""
|
|
||||||
inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)
|
|
||||||
|
|
||||||
if prompt is not None:
|
|
||||||
inputs["prompt"] = prompt
|
|
||||||
if cache_salt is not None:
|
|
||||||
inputs["cache_salt"] = cache_salt
|
|
||||||
|
|
||||||
return inputs
|
|
||||||
|
|
||||||
|
|
||||||
class EmbedsInputs(_InputOptions):
|
|
||||||
"""Represents embeddings-based inputs."""
|
|
||||||
|
|
||||||
type: Literal["embeds"]
|
|
||||||
"""The type of inputs."""
|
|
||||||
|
|
||||||
prompt_embeds: torch.Tensor
|
|
||||||
"""The embeddings of the prompt."""
|
|
||||||
|
|
||||||
prompt: NotRequired[str]
|
|
||||||
"""The prompt text corresponding to the token IDs, if available."""
|
|
||||||
|
|
||||||
|
|
||||||
def embeds_inputs(
|
|
||||||
prompt_embeds: torch.Tensor,
|
|
||||||
*,
|
|
||||||
prompt: str | None = None,
|
|
||||||
cache_salt: str | None = None,
|
|
||||||
) -> EmbedsInputs:
|
|
||||||
"""Construct [`EmbedsInputs`][vllm.inputs.data.EmbedsInputs] from optional
|
|
||||||
values."""
|
|
||||||
inputs = EmbedsInputs(type="embeds", prompt_embeds=prompt_embeds)
|
|
||||||
|
|
||||||
if prompt is not None:
|
|
||||||
inputs["prompt"] = prompt
|
|
||||||
if cache_salt is not None:
|
|
||||||
inputs["cache_salt"] = cache_salt
|
|
||||||
|
|
||||||
return inputs
|
|
||||||
|
|
||||||
|
|
||||||
DecoderOnlyInputs: TypeAlias = TokenInputs | EmbedsInputs | MultiModalInputs
|
|
||||||
"""
|
|
||||||
A processed prompt from
|
|
||||||
[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]
|
|
||||||
which can be passed to
|
|
||||||
[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]
|
|
||||||
for decoder-only models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
EncoderInputs: TypeAlias = TokenInputs | MultiModalEncDecInputs
|
|
||||||
"""
|
|
||||||
A processed encoder prompt from
|
|
||||||
[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]
|
|
||||||
which can be passed to
|
|
||||||
[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]
|
|
||||||
for encoder-decoder models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
DecoderInputs: TypeAlias = TokenInputs | MultiModalInputs
|
|
||||||
"""
|
|
||||||
A processed decoder prompt from
|
|
||||||
[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]
|
|
||||||
which can be passed to
|
|
||||||
[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]
|
|
||||||
for encoder-decoder models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class EncoderDecoderInputs(TypedDict):
|
|
||||||
"""
|
|
||||||
A processed pair of encoder and decoder singleton prompts.
|
|
||||||
[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]
|
|
||||||
which can be passed to
|
|
||||||
[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]
|
|
||||||
for encoder-decoder models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
type: Literal["enc_dec"]
|
|
||||||
|
|
||||||
encoder_prompt: EncoderInputs
|
|
||||||
"""The inputs for the encoder portion."""
|
|
||||||
|
|
||||||
decoder_prompt: DecoderInputs
|
|
||||||
"""The inputs for the decoder portion."""
|
|
||||||
|
|
||||||
arrival_time: NotRequired[float]
|
|
||||||
"""The time when the input was received (before rendering)."""
|
|
||||||
|
|
||||||
|
|
||||||
ProcessorInputs: TypeAlias = DecoderOnlyInputs | EncoderDecoderInputs
|
|
||||||
"""
|
|
||||||
A processed prompt from
|
|
||||||
[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]
|
|
||||||
which can be passed to
|
|
||||||
[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor].
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
SingletonInputs: TypeAlias = DecoderOnlyInputs | MultiModalEncDecInputs
|
|
||||||
"""The inputs for a single encoder/decoder prompt."""
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_enc_inputs(inputs: SingletonInputs) -> EncoderInputs:
|
|
||||||
if inputs["type"] == "embeds":
|
|
||||||
raise ValueError(
|
|
||||||
"Embedding inputs are not supported for encoder-decoder models"
|
|
||||||
)
|
|
||||||
|
|
||||||
if inputs["type"] == "multimodal" and "encoder_prompt_token_ids" not in inputs:
|
|
||||||
raise RuntimeError(
|
|
||||||
"You should register an encoder-decoder multi-modal processor "
|
|
||||||
"for encoder-decoder models."
|
|
||||||
)
|
|
||||||
|
|
||||||
return inputs # type: ignore[return-value]
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_dec_inputs(inputs: SingletonInputs) -> DecoderInputs:
|
|
||||||
if inputs["type"] == "embeds":
|
|
||||||
raise ValueError(
|
|
||||||
"Embedding inputs are not supported for encoder-decoder models"
|
|
||||||
)
|
|
||||||
|
|
||||||
return inputs
|
|
||||||
|
|
||||||
|
|
||||||
def _prepare_decoder_input_ids_for_generation(
|
|
||||||
decoder_input_ids: list[int],
|
|
||||||
decoder_start_token_id: int,
|
|
||||||
) -> list[int]:
|
|
||||||
"""
|
|
||||||
Prepare `decoder_input_ids` for generation with encoder-decoder models,
|
|
||||||
according to `GenerationMixin._prepare_decoder_input_ids_for_generation()`.
|
|
||||||
|
|
||||||
Source:
|
|
||||||
https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/generation/utils.py
|
|
||||||
"""
|
|
||||||
if len(decoder_input_ids) == 0 or decoder_input_ids[0] != decoder_start_token_id:
|
|
||||||
decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
|
|
||||||
|
|
||||||
return decoder_input_ids
|
|
||||||
|
|
||||||
|
|
||||||
def build_enc_dec_inputs(
|
|
||||||
encoder_inputs: SingletonInputs,
|
|
||||||
decoder_inputs: SingletonInputs | None,
|
|
||||||
decoder_start_token_id: int,
|
|
||||||
skip_decoder_start_token: bool = False,
|
|
||||||
) -> EncoderDecoderInputs:
|
|
||||||
enc_inputs = _validate_enc_inputs(encoder_inputs)
|
|
||||||
|
|
||||||
if decoder_inputs is None:
|
|
||||||
dec_inputs: DecoderInputs = enc_inputs
|
|
||||||
else:
|
|
||||||
dec_inputs = _validate_dec_inputs(decoder_inputs)
|
|
||||||
|
|
||||||
enc_inputs_new: EncoderInputs
|
|
||||||
dec_inputs_new: DecoderInputs
|
|
||||||
|
|
||||||
if enc_inputs["type"] == "multimodal":
|
|
||||||
from vllm.multimodal.inputs import mm_inputs
|
|
||||||
|
|
||||||
enc_inputs_new = token_inputs(
|
|
||||||
enc_inputs["encoder_prompt_token_ids"],
|
|
||||||
prompt=enc_inputs.get("encoder_prompt"),
|
|
||||||
)
|
|
||||||
dec_inputs_new = mm_inputs(
|
|
||||||
prompt_token_ids=dec_inputs["prompt_token_ids"],
|
|
||||||
prompt=dec_inputs.get("prompt"),
|
|
||||||
mm_kwargs=enc_inputs["mm_kwargs"],
|
|
||||||
mm_hashes=enc_inputs["mm_hashes"],
|
|
||||||
mm_placeholders=enc_inputs["mm_placeholders"],
|
|
||||||
)
|
|
||||||
elif enc_inputs["type"] == "token":
|
|
||||||
enc_inputs_new = token_inputs(prompt_token_ids=[])
|
|
||||||
dec_inputs_new = dec_inputs
|
|
||||||
else:
|
|
||||||
assert_never(enc_inputs)
|
|
||||||
|
|
||||||
if not skip_decoder_start_token:
|
|
||||||
dec_inputs_new["prompt_token_ids"] = _prepare_decoder_input_ids_for_generation(
|
|
||||||
dec_inputs_new["prompt_token_ids"],
|
|
||||||
decoder_start_token_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
if cache_salt := enc_inputs.get("cache_salt"):
|
|
||||||
dec_inputs_new["cache_salt"] = cache_salt
|
|
||||||
|
|
||||||
return EncoderDecoderInputs(
|
|
||||||
type="enc_dec",
|
|
||||||
encoder_prompt=enc_inputs_new,
|
|
||||||
decoder_prompt=dec_inputs_new,
|
|
||||||
)
|
|
||||||
@@ -0,0 +1,352 @@
|
|||||||
|
"""Schema and utilities for inputs to the engine client (`LLMEngine`/`AsyncLLM`)."""
|
||||||
|
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from typing import TYPE_CHECKING, Literal, TypeAlias
|
||||||
|
|
||||||
|
from typing_extensions import NotRequired, TypedDict, assert_never
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems, PlaceholderRange
|
||||||
|
|
||||||
|
|
||||||
|
class _InputOptions(TypedDict):
|
||||||
|
"""
|
||||||
|
Additional options available to all
|
||||||
|
[`SingletonInput`][vllm.inputs.engine.SingletonInput] types.
|
||||||
|
"""
|
||||||
|
|
||||||
|
arrival_time: NotRequired[float]
|
||||||
|
"""The time when the input was received (before rendering)."""
|
||||||
|
|
||||||
|
cache_salt: NotRequired[str]
|
||||||
|
"""Optional cache salt to be used for prefix caching."""
|
||||||
|
|
||||||
|
|
||||||
|
class TokensInput(_InputOptions):
|
||||||
|
"""Represents token-based input to the engine."""
|
||||||
|
|
||||||
|
type: Literal["token"]
|
||||||
|
"""The type of input."""
|
||||||
|
|
||||||
|
prompt_token_ids: list[int]
|
||||||
|
"""The token IDs of the prompt."""
|
||||||
|
|
||||||
|
prompt: NotRequired[str]
|
||||||
|
"""The prompt text corresponding to the token IDs, if available."""
|
||||||
|
|
||||||
|
|
||||||
|
def tokens_input(
|
||||||
|
prompt_token_ids: list[int],
|
||||||
|
*,
|
||||||
|
prompt: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
|
) -> TokensInput:
|
||||||
|
"""
|
||||||
|
Construct [`TokensInput`][vllm.inputs.engine.TokensInput]
|
||||||
|
from optional values.
|
||||||
|
"""
|
||||||
|
inputs = TokensInput(type="token", prompt_token_ids=prompt_token_ids)
|
||||||
|
|
||||||
|
if prompt is not None:
|
||||||
|
inputs["prompt"] = prompt
|
||||||
|
if cache_salt is not None:
|
||||||
|
inputs["cache_salt"] = cache_salt
|
||||||
|
|
||||||
|
return inputs
|
||||||
|
|
||||||
|
|
||||||
|
class EmbedsInput(_InputOptions):
|
||||||
|
"""Represents embeddings-based input to the engine."""
|
||||||
|
|
||||||
|
type: Literal["embeds"]
|
||||||
|
"""The type of input."""
|
||||||
|
|
||||||
|
prompt_embeds: "torch.Tensor"
|
||||||
|
"""The embeddings of the prompt."""
|
||||||
|
|
||||||
|
prompt: NotRequired[str]
|
||||||
|
"""The prompt text corresponding to the token IDs, if available."""
|
||||||
|
|
||||||
|
|
||||||
|
def embeds_input(
|
||||||
|
prompt_embeds: "torch.Tensor",
|
||||||
|
*,
|
||||||
|
prompt: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
|
) -> EmbedsInput:
|
||||||
|
"""
|
||||||
|
Construct [`EmbedsInput`][vllm.inputs.engine.EmbedsInput]
|
||||||
|
from optional values.
|
||||||
|
"""
|
||||||
|
inputs = EmbedsInput(type="embeds", prompt_embeds=prompt_embeds)
|
||||||
|
|
||||||
|
if prompt is not None:
|
||||||
|
inputs["prompt"] = prompt
|
||||||
|
if cache_salt is not None:
|
||||||
|
inputs["cache_salt"] = cache_salt
|
||||||
|
|
||||||
|
return inputs
|
||||||
|
|
||||||
|
|
||||||
|
MultiModalHashes: TypeAlias = Mapping[str, list[str]]
|
||||||
|
"""
|
||||||
|
A dictionary containing per-item hashes for each modality.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
MultiModalPlaceholders: TypeAlias = Mapping[str, Sequence["PlaceholderRange"]]
|
||||||
|
"""
|
||||||
|
A dictionary containing per-item placeholder ranges for each modality.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class MultiModalInput(_InputOptions):
|
||||||
|
"""Represents multi-modal input to the engine."""
|
||||||
|
|
||||||
|
type: Literal["multimodal"]
|
||||||
|
"""The type of input."""
|
||||||
|
|
||||||
|
prompt_token_ids: list[int]
|
||||||
|
"""The processed token IDs which includes placeholder tokens."""
|
||||||
|
|
||||||
|
prompt: NotRequired[str]
|
||||||
|
"""The prompt text corresponding to the token IDs, if available."""
|
||||||
|
|
||||||
|
mm_kwargs: "MultiModalKwargsOptionalItems"
|
||||||
|
"""Keyword arguments to be directly passed to the model after batching."""
|
||||||
|
|
||||||
|
mm_hashes: MultiModalHashes
|
||||||
|
"""The hashes of the multi-modal data."""
|
||||||
|
|
||||||
|
mm_placeholders: MultiModalPlaceholders
|
||||||
|
"""
|
||||||
|
For each modality, information about the placeholder tokens in
|
||||||
|
`prompt_token_ids`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def mm_input(
|
||||||
|
prompt_token_ids: list[int],
|
||||||
|
mm_kwargs: "MultiModalKwargsOptionalItems",
|
||||||
|
mm_hashes: MultiModalHashes,
|
||||||
|
mm_placeholders: MultiModalPlaceholders,
|
||||||
|
*,
|
||||||
|
prompt: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
|
) -> MultiModalInput:
|
||||||
|
inputs = MultiModalInput(
|
||||||
|
type="multimodal",
|
||||||
|
prompt_token_ids=prompt_token_ids,
|
||||||
|
mm_kwargs=mm_kwargs,
|
||||||
|
mm_hashes=mm_hashes,
|
||||||
|
mm_placeholders=mm_placeholders,
|
||||||
|
)
|
||||||
|
|
||||||
|
if prompt is not None:
|
||||||
|
inputs["prompt"] = prompt
|
||||||
|
if cache_salt is not None:
|
||||||
|
inputs["cache_salt"] = cache_salt
|
||||||
|
|
||||||
|
return inputs
|
||||||
|
|
||||||
|
|
||||||
|
class MultiModalEncDecInput(MultiModalInput):
|
||||||
|
"""
|
||||||
|
Represents multi-modal input to the engine for encoder-decoder models.
|
||||||
|
|
||||||
|
Note:
|
||||||
|
Even text-only encoder-decoder models are currently implemented
|
||||||
|
as multi-modal models for convenience.
|
||||||
|
(Example: https://github.com/vllm-project/bart-plugin)
|
||||||
|
"""
|
||||||
|
|
||||||
|
encoder_prompt_token_ids: list[int]
|
||||||
|
"""The processed token IDs of the encoder prompt."""
|
||||||
|
|
||||||
|
encoder_prompt: NotRequired[str]
|
||||||
|
"""The prompt text corresponding to the encoder token IDs, if available."""
|
||||||
|
|
||||||
|
|
||||||
|
def mm_enc_dec_input(
|
||||||
|
encoder_inputs: MultiModalInput,
|
||||||
|
decoder_prompt_token_ids: list[int],
|
||||||
|
*,
|
||||||
|
decoder_prompt: str | None = None,
|
||||||
|
) -> MultiModalEncDecInput:
|
||||||
|
inputs = MultiModalEncDecInput(
|
||||||
|
type="multimodal",
|
||||||
|
prompt_token_ids=decoder_prompt_token_ids,
|
||||||
|
encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
|
||||||
|
mm_kwargs=encoder_inputs["mm_kwargs"],
|
||||||
|
mm_hashes=encoder_inputs["mm_hashes"],
|
||||||
|
mm_placeholders=encoder_inputs["mm_placeholders"],
|
||||||
|
)
|
||||||
|
|
||||||
|
if decoder_prompt is not None:
|
||||||
|
inputs["prompt"] = decoder_prompt
|
||||||
|
if "prompt" in encoder_inputs:
|
||||||
|
inputs["encoder_prompt"] = encoder_inputs["prompt"]
|
||||||
|
if "cache_salt" in encoder_inputs:
|
||||||
|
inputs["cache_salt"] = encoder_inputs["cache_salt"]
|
||||||
|
|
||||||
|
return inputs
|
||||||
|
|
||||||
|
|
||||||
|
DecoderOnlyEngineInput: TypeAlias = TokensInput | EmbedsInput | MultiModalInput
|
||||||
|
"""
|
||||||
|
A rendered [`DecoderOnlyPrompt`][vllm.inputs.llm.DecoderOnlyPrompt]
|
||||||
|
which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
EncoderInput: TypeAlias = TokensInput | MultiModalEncDecInput
|
||||||
|
"""
|
||||||
|
A rendered [`EncoderPrompt`][vllm.inputs.llm.EncoderPrompt]
|
||||||
|
which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
DecoderEngineInput: TypeAlias = TokensInput | MultiModalInput
|
||||||
|
"""
|
||||||
|
A rendered [`DecoderPrompt`][vllm.inputs.llm.DecoderPrompt]
|
||||||
|
which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class EncoderDecoderInput(TypedDict):
|
||||||
|
"""
|
||||||
|
A rendered [`EncoderDecoderPrompt`][vllm.inputs.llm.EncoderDecoderPrompt]
|
||||||
|
which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
type: Literal["enc_dec"]
|
||||||
|
|
||||||
|
encoder_prompt: EncoderInput
|
||||||
|
"""The inputs for the encoder portion."""
|
||||||
|
|
||||||
|
decoder_prompt: DecoderEngineInput
|
||||||
|
"""The inputs for the decoder portion."""
|
||||||
|
|
||||||
|
arrival_time: NotRequired[float]
|
||||||
|
"""The time when the input was received (before rendering)."""
|
||||||
|
|
||||||
|
|
||||||
|
SingletonInput: TypeAlias = DecoderOnlyEngineInput | MultiModalEncDecInput
|
||||||
|
"""
|
||||||
|
A rendered [`SingletonPrompt`][vllm.inputs.llm.SingletonPrompt]
|
||||||
|
which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
EngineInput: TypeAlias = DecoderOnlyEngineInput | EncoderDecoderInput
|
||||||
|
"""
|
||||||
|
A rendered [`PromptType`][vllm.inputs.llm.PromptType]
|
||||||
|
which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_enc_input(enc_input: SingletonInput) -> EncoderInput:
|
||||||
|
if enc_input["type"] == "embeds":
|
||||||
|
raise ValueError(
|
||||||
|
"Embedding inputs are not supported for encoder-decoder models"
|
||||||
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
enc_input["type"] == "multimodal"
|
||||||
|
and "encoder_prompt_token_ids" not in enc_input
|
||||||
|
):
|
||||||
|
raise RuntimeError(
|
||||||
|
"You should register an encoder-decoder multi-modal processor "
|
||||||
|
"for encoder-decoder models."
|
||||||
|
)
|
||||||
|
|
||||||
|
return enc_input # type: ignore[return-value]
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_dec_input(dec_input: SingletonInput) -> DecoderEngineInput:
|
||||||
|
if dec_input["type"] == "embeds":
|
||||||
|
raise ValueError(
|
||||||
|
"Embedding inputs are not supported for encoder-decoder models"
|
||||||
|
)
|
||||||
|
|
||||||
|
return dec_input
|
||||||
|
|
||||||
|
|
||||||
|
def _prepare_decoder_input_ids_for_generation(
|
||||||
|
decoder_input_ids: list[int],
|
||||||
|
decoder_start_token_id: int,
|
||||||
|
) -> list[int]:
|
||||||
|
"""
|
||||||
|
Prepare `decoder_input_ids` for generation with encoder-decoder models,
|
||||||
|
according to `GenerationMixin._prepare_decoder_input_ids_for_generation()`.
|
||||||
|
|
||||||
|
Source:
|
||||||
|
https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/generation/utils.py
|
||||||
|
"""
|
||||||
|
if len(decoder_input_ids) == 0 or decoder_input_ids[0] != decoder_start_token_id:
|
||||||
|
decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
|
||||||
|
|
||||||
|
return decoder_input_ids
|
||||||
|
|
||||||
|
|
||||||
|
def build_enc_dec_input(
|
||||||
|
encoder_input: SingletonInput,
|
||||||
|
decoder_input: SingletonInput | None,
|
||||||
|
decoder_start_token_id: int,
|
||||||
|
skip_decoder_start_token: bool = False,
|
||||||
|
) -> EncoderDecoderInput:
|
||||||
|
enc_input = _validate_enc_input(encoder_input)
|
||||||
|
|
||||||
|
if decoder_input is None:
|
||||||
|
dec_input: DecoderEngineInput = enc_input
|
||||||
|
else:
|
||||||
|
dec_input = _validate_dec_input(decoder_input)
|
||||||
|
|
||||||
|
enc_input_new: EncoderInput
|
||||||
|
dec_input_new: DecoderEngineInput
|
||||||
|
|
||||||
|
if enc_input["type"] == "multimodal":
|
||||||
|
enc_input_new = tokens_input(
|
||||||
|
enc_input["encoder_prompt_token_ids"],
|
||||||
|
prompt=enc_input.get("encoder_prompt"),
|
||||||
|
)
|
||||||
|
dec_input_new = mm_input(
|
||||||
|
prompt_token_ids=dec_input["prompt_token_ids"],
|
||||||
|
prompt=dec_input.get("prompt"),
|
||||||
|
mm_kwargs=enc_input["mm_kwargs"],
|
||||||
|
mm_hashes=enc_input["mm_hashes"],
|
||||||
|
mm_placeholders=enc_input["mm_placeholders"],
|
||||||
|
)
|
||||||
|
elif enc_input["type"] == "token":
|
||||||
|
enc_input_new = tokens_input(prompt_token_ids=[])
|
||||||
|
dec_input_new = dec_input
|
||||||
|
else:
|
||||||
|
assert_never(enc_input)
|
||||||
|
|
||||||
|
if not skip_decoder_start_token:
|
||||||
|
dec_input_new["prompt_token_ids"] = _prepare_decoder_input_ids_for_generation(
|
||||||
|
dec_input_new["prompt_token_ids"],
|
||||||
|
decoder_start_token_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
if cache_salt := enc_input.get("cache_salt"):
|
||||||
|
dec_input_new["cache_salt"] = cache_salt
|
||||||
|
|
||||||
|
return EncoderDecoderInput(
|
||||||
|
type="enc_dec",
|
||||||
|
encoder_prompt=enc_input_new,
|
||||||
|
decoder_prompt=dec_input_new,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def split_enc_dec_input(
|
||||||
|
inputs: EngineInput,
|
||||||
|
) -> tuple[SingletonInput | None, SingletonInput]:
|
||||||
|
if inputs["type"] == "enc_dec":
|
||||||
|
return inputs["encoder_prompt"], inputs["decoder_prompt"]
|
||||||
|
|
||||||
|
return None, inputs
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
"""Schema and utilities for input prompts to the LLM API."""
|
||||||
|
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, final
|
||||||
|
|
||||||
|
from typing_extensions import NotRequired, TypedDict
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.multimodal.inputs import AudioItem, ImageItem, VideoItem, VisionChunk
|
||||||
|
|
||||||
|
|
||||||
|
_T = TypeVar("_T")
|
||||||
|
|
||||||
|
ModalityData: TypeAlias = _T | list[_T | None] | None
|
||||||
|
"""
|
||||||
|
Either a single data item, or a list of data items. Can only be None if UUID
|
||||||
|
is provided.
|
||||||
|
|
||||||
|
The number of data items allowed per modality is restricted by
|
||||||
|
`--limit-mm-per-prompt`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
@final
|
||||||
|
class MultiModalDataBuiltins(TypedDict, total=False):
|
||||||
|
"""Type annotations for modality types predefined by vLLM."""
|
||||||
|
|
||||||
|
image: ModalityData["ImageItem"]
|
||||||
|
"""The input image(s)."""
|
||||||
|
|
||||||
|
video: ModalityData["VideoItem"]
|
||||||
|
"""The input video(s)."""
|
||||||
|
|
||||||
|
audio: ModalityData["AudioItem"]
|
||||||
|
"""The input audio(s)."""
|
||||||
|
|
||||||
|
vision_chunk: ModalityData["VisionChunk"]
|
||||||
|
"""The input visual atom(s) - unified modality for images and video chunks."""
|
||||||
|
|
||||||
|
|
||||||
|
MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]
|
||||||
|
"""
|
||||||
|
A dictionary containing an entry for each modality type to input.
|
||||||
|
|
||||||
|
The built-in modalities are defined by
|
||||||
|
[`MultiModalDataBuiltins`][vllm.inputs.llm.MultiModalDataBuiltins].
|
||||||
|
"""
|
||||||
|
|
||||||
|
MultiModalUUIDDict: TypeAlias = Mapping[str, Sequence[str | None] | str]
|
||||||
|
"""
|
||||||
|
A dictionary containing user-provided UUIDs for items in each modality.
|
||||||
|
If a UUID for an item is not provided, its entry will be `None` and
|
||||||
|
MultiModalHasher will compute a hash for the item.
|
||||||
|
|
||||||
|
The UUID will be used to identify the item for all caching purposes
|
||||||
|
(input processing caching, embedding caching, prefix caching, etc).
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class _PromptOptions(TypedDict):
|
||||||
|
"""
|
||||||
|
Additional options available to all
|
||||||
|
[`SingletonPrompt`][vllm.inputs.llm.SingletonPrompt] types.
|
||||||
|
"""
|
||||||
|
|
||||||
|
multi_modal_data: NotRequired[MultiModalDataDict | None]
|
||||||
|
"""
|
||||||
|
Optional multi-modal data to pass to the model,
|
||||||
|
if the model supports it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
mm_processor_kwargs: NotRequired[dict[str, Any] | None]
|
||||||
|
"""
|
||||||
|
Optional multi-modal processor kwargs to be forwarded to the
|
||||||
|
multimodal input mapper & processor. Note that if multiple modalities
|
||||||
|
have registered mappers etc for the model being considered, we attempt
|
||||||
|
to pass the mm_processor_kwargs to each of them.
|
||||||
|
"""
|
||||||
|
|
||||||
|
multi_modal_uuids: NotRequired[MultiModalUUIDDict]
|
||||||
|
"""
|
||||||
|
Optional user-specified UUIDs for multimodal items, mapped by modality.
|
||||||
|
Lists must match the number of items per modality and may contain `None`.
|
||||||
|
For `None` entries, the hasher will compute IDs automatically; non-None
|
||||||
|
entries override the default hashes for caching, and MUST be unique per
|
||||||
|
multimodal item.
|
||||||
|
"""
|
||||||
|
|
||||||
|
cache_salt: NotRequired[str]
|
||||||
|
"""
|
||||||
|
Optional cache salt to be used for prefix caching.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TextPrompt(_PromptOptions):
|
||||||
|
"""Schema for a text prompt."""
|
||||||
|
|
||||||
|
prompt: str
|
||||||
|
"""The input text to be tokenized before passing to the model."""
|
||||||
|
|
||||||
|
|
||||||
|
class TokensPrompt(_PromptOptions):
|
||||||
|
"""Schema for a tokenized prompt."""
|
||||||
|
|
||||||
|
prompt_token_ids: list[int]
|
||||||
|
"""A list of token IDs to pass to the model."""
|
||||||
|
|
||||||
|
prompt: NotRequired[str]
|
||||||
|
"""The prompt text corresponding to the token IDs, if available."""
|
||||||
|
|
||||||
|
token_type_ids: NotRequired[list[int]]
|
||||||
|
"""A list of token type IDs to pass to the cross encoder model."""
|
||||||
|
|
||||||
|
|
||||||
|
class EmbedsPrompt(_PromptOptions):
|
||||||
|
"""Schema for a prompt provided via token embeddings."""
|
||||||
|
|
||||||
|
prompt_embeds: "torch.Tensor"
|
||||||
|
"""The embeddings of the prompt."""
|
||||||
|
|
||||||
|
prompt: NotRequired[str]
|
||||||
|
"""The prompt text corresponding to the token embeddings, if available."""
|
||||||
|
|
||||||
|
|
||||||
|
DecoderOnlyPrompt: TypeAlias = (
|
||||||
|
str | TextPrompt | list[int] | TokensPrompt | EmbedsPrompt
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
Schema of a prompt for a decoder-only model:
|
||||||
|
|
||||||
|
- A text prompt (string or [`TextPrompt`][vllm.inputs.llm.TextPrompt])
|
||||||
|
- A tokenized prompt (list of token IDs, or
|
||||||
|
[`TokensPrompt`][vllm.inputs.llm.TokensPrompt])
|
||||||
|
- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.llm.EmbedsPrompt])
|
||||||
|
|
||||||
|
For encoder-decoder models, passing a singleton prompt is shorthand for passing
|
||||||
|
`ExplicitEncoderDecoderPrompt(encoder_prompt=prompt, decoder_prompt=None)`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
EncoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt
|
||||||
|
"""
|
||||||
|
Schema of a prompt for the encoder part of a encoder-decoder model:
|
||||||
|
|
||||||
|
- A text prompt (string or [`TextPrompt`][vllm.inputs.llm.TextPrompt])
|
||||||
|
- A tokenized prompt (list of token IDs, or
|
||||||
|
[`TokensPrompt`][vllm.inputs.llm.TokensPrompt])
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
DecoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt
|
||||||
|
"""
|
||||||
|
Schema of a prompt for the decoder part of an encoder-decoder model:
|
||||||
|
|
||||||
|
- A text prompt (string or [`TextPrompt`][vllm.inputs.llm.TextPrompt])
|
||||||
|
- A tokenized prompt (list of token IDs, or
|
||||||
|
[`TokensPrompt`][vllm.inputs.llm.TokensPrompt])
|
||||||
|
|
||||||
|
Note:
|
||||||
|
Multi-modal inputs are not supported for decoder prompts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class ExplicitEncoderDecoderPrompt(TypedDict):
|
||||||
|
"""
|
||||||
|
Schema for a pair of encoder and decoder singleton prompts.
|
||||||
|
|
||||||
|
Note:
|
||||||
|
This schema is not valid for decoder-only models.
|
||||||
|
"""
|
||||||
|
|
||||||
|
encoder_prompt: EncoderPrompt
|
||||||
|
"""The prompt for the encoder part of the model."""
|
||||||
|
|
||||||
|
decoder_prompt: DecoderPrompt | None
|
||||||
|
"""
|
||||||
|
The prompt for the decoder part of the model.
|
||||||
|
|
||||||
|
Passing `None` will cause the prompt to be inferred automatically.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
EncoderDecoderPrompt: TypeAlias = EncoderPrompt | ExplicitEncoderDecoderPrompt
|
||||||
|
"""
|
||||||
|
Schema for a prompt for an encoder-decoder model.
|
||||||
|
|
||||||
|
You can pass a singleton encoder prompt, in which case the decoder prompt is
|
||||||
|
considered to be `None` (i.e., infer automatically).
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
SingletonPrompt: TypeAlias = DecoderOnlyPrompt | EncoderPrompt | DecoderPrompt
|
||||||
|
"""
|
||||||
|
Schema for a single prompt. This is as opposed to a data structure
|
||||||
|
which encapsulates multiple prompts, such as
|
||||||
|
[`ExplicitEncoderDecoderPrompt`][vllm.inputs.llm.ExplicitEncoderDecoderPrompt].
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
PromptType: TypeAlias = DecoderOnlyPrompt | EncoderDecoderPrompt
|
||||||
|
"""
|
||||||
|
Schema for any prompt, regardless of model type.
|
||||||
|
|
||||||
|
This is the input format accepted by most [`LLM`][vllm.entrypoints.llm.LLM] APIs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class DataPrompt(_PromptOptions):
|
||||||
|
"""
|
||||||
|
Represents generic inputs that are converted to
|
||||||
|
[`PromptType`][vllm.inputs.llm.PromptType] by IO processor plugins.
|
||||||
|
"""
|
||||||
|
|
||||||
|
data: Any
|
||||||
|
"""The input data."""
|
||||||
|
|
||||||
|
data_format: str
|
||||||
|
"""The input data format."""
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from .data import ProcessorInputs, SingletonInputs
|
|
||||||
|
|
||||||
|
|
||||||
def split_enc_dec_inputs(
|
|
||||||
inputs: ProcessorInputs,
|
|
||||||
) -> tuple[SingletonInputs | None, SingletonInputs]:
|
|
||||||
if inputs["type"] == "enc_dec":
|
|
||||||
return inputs["encoder_prompt"], inputs["decoder_prompt"]
|
|
||||||
|
|
||||||
return None, inputs
|
|
||||||
+34
-73
@@ -7,14 +7,9 @@ from typing import Any, overload
|
|||||||
from typing_extensions import assert_never
|
from typing_extensions import assert_never
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.inputs.data import build_enc_dec_inputs
|
from vllm.inputs import build_enc_dec_input
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||||
from vllm.multimodal.inputs import (
|
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalInputs,
|
|
||||||
MultiModalUUIDDict,
|
|
||||||
)
|
|
||||||
from vllm.renderers import BaseRenderer, renderer_from_config
|
from vllm.renderers import BaseRenderer, renderer_from_config
|
||||||
from vllm.renderers.inputs import (
|
from vllm.renderers.inputs import (
|
||||||
DecoderDictPrompt,
|
DecoderDictPrompt,
|
||||||
@@ -26,20 +21,25 @@ from vllm.renderers.inputs import (
|
|||||||
from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt
|
from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
|
||||||
from .data import (
|
from .engine import (
|
||||||
DecoderInputs,
|
DecoderEngineInput,
|
||||||
DecoderOnlyInputs,
|
DecoderOnlyEngineInput,
|
||||||
EmbedsInputs,
|
EmbedsInput,
|
||||||
|
EncoderDecoderInput,
|
||||||
|
EncoderInput,
|
||||||
|
EngineInput,
|
||||||
|
MultiModalInput,
|
||||||
|
SingletonInput,
|
||||||
|
TokensInput,
|
||||||
|
tokens_input,
|
||||||
|
)
|
||||||
|
from .llm import (
|
||||||
EmbedsPrompt,
|
EmbedsPrompt,
|
||||||
EncoderDecoderInputs,
|
MultiModalDataDict,
|
||||||
EncoderInputs,
|
MultiModalUUIDDict,
|
||||||
ProcessorInputs,
|
|
||||||
PromptType,
|
PromptType,
|
||||||
SingletonInputs,
|
|
||||||
TextPrompt,
|
TextPrompt,
|
||||||
TokenInputs,
|
|
||||||
TokensPrompt,
|
TokensPrompt,
|
||||||
token_inputs,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
@@ -95,7 +95,7 @@ class InputPreprocessor:
|
|||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
*,
|
*,
|
||||||
mm_uuids: MultiModalUUIDDict | None = None,
|
mm_uuids: MultiModalUUIDDict | None = None,
|
||||||
) -> MultiModalInputs:
|
) -> MultiModalInput:
|
||||||
"""
|
"""
|
||||||
Apply the model's multi-modal processor to a multi-modal prompt,
|
Apply the model's multi-modal processor to a multi-modal prompt,
|
||||||
returning the corresponding token IDs and metadata.
|
returning the corresponding token IDs and metadata.
|
||||||
@@ -111,7 +111,7 @@ class InputPreprocessor:
|
|||||||
def _process_embeds(
|
def _process_embeds(
|
||||||
self,
|
self,
|
||||||
parsed_content: EmbedsPrompt,
|
parsed_content: EmbedsPrompt,
|
||||||
) -> EmbedsInputs:
|
) -> EmbedsInput:
|
||||||
return self.renderer._process_embeds(parsed_content)
|
return self.renderer._process_embeds(parsed_content)
|
||||||
|
|
||||||
def _truncate_inputs(
|
def _truncate_inputs(
|
||||||
@@ -134,12 +134,12 @@ class InputPreprocessor:
|
|||||||
self,
|
self,
|
||||||
parsed_content: TokensPrompt,
|
parsed_content: TokensPrompt,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> TokenInputs | MultiModalInputs:
|
) -> TokensInput | MultiModalInput:
|
||||||
prompt_token_ids = self._truncate_inputs(
|
prompt_token_ids = self._truncate_inputs(
|
||||||
parsed_content["prompt_token_ids"], tokenization_kwargs
|
parsed_content["prompt_token_ids"], tokenization_kwargs
|
||||||
)
|
)
|
||||||
|
|
||||||
inputs: TokenInputs | MultiModalInputs
|
inputs: TokensInput | MultiModalInput
|
||||||
if multi_modal_data := parsed_content.get("multi_modal_data"):
|
if multi_modal_data := parsed_content.get("multi_modal_data"):
|
||||||
inputs = self._process_multimodal(
|
inputs = self._process_multimodal(
|
||||||
prompt_token_ids,
|
prompt_token_ids,
|
||||||
@@ -149,7 +149,7 @@ class InputPreprocessor:
|
|||||||
mm_uuids=parsed_content.get("multi_modal_uuids"),
|
mm_uuids=parsed_content.get("multi_modal_uuids"),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
inputs = token_inputs(prompt_token_ids)
|
inputs = tokens_input(prompt_token_ids)
|
||||||
|
|
||||||
if prompt_text := parsed_content.get("prompt"):
|
if prompt_text := parsed_content.get("prompt"):
|
||||||
inputs["prompt"] = prompt_text
|
inputs["prompt"] = prompt_text
|
||||||
@@ -162,10 +162,10 @@ class InputPreprocessor:
|
|||||||
self,
|
self,
|
||||||
parsed_content: TextPrompt,
|
parsed_content: TextPrompt,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> TokenInputs | MultiModalInputs:
|
) -> TokensInput | MultiModalInput:
|
||||||
prompt_text = parsed_content["prompt"]
|
prompt_text = parsed_content["prompt"]
|
||||||
|
|
||||||
inputs: TokenInputs | MultiModalInputs
|
inputs: TokensInput | MultiModalInput
|
||||||
if multi_modal_data := parsed_content.get("multi_modal_data"):
|
if multi_modal_data := parsed_content.get("multi_modal_data"):
|
||||||
inputs = self._process_multimodal(
|
inputs = self._process_multimodal(
|
||||||
prompt_text,
|
prompt_text,
|
||||||
@@ -178,7 +178,7 @@ class InputPreprocessor:
|
|||||||
prompt_text,
|
prompt_text,
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
tokenization_kwargs=tokenization_kwargs,
|
||||||
)
|
)
|
||||||
inputs = token_inputs(prompt_token_ids)
|
inputs = tokens_input(prompt_token_ids)
|
||||||
|
|
||||||
inputs["prompt"] = prompt_text
|
inputs["prompt"] = prompt_text
|
||||||
|
|
||||||
@@ -192,38 +192,27 @@ class InputPreprocessor:
|
|||||||
self,
|
self,
|
||||||
prompt: EncoderDictPrompt,
|
prompt: EncoderDictPrompt,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> EncoderInputs: ...
|
) -> EncoderInput: ...
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def _prompt_to_llm_inputs( # type: ignore[misc]
|
def _prompt_to_llm_inputs( # type: ignore[misc]
|
||||||
self,
|
self,
|
||||||
prompt: DecoderDictPrompt,
|
prompt: DecoderDictPrompt,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> DecoderInputs: ...
|
) -> DecoderEngineInput: ...
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def _prompt_to_llm_inputs( # type: ignore[misc]
|
def _prompt_to_llm_inputs( # type: ignore[misc]
|
||||||
self,
|
self,
|
||||||
prompt: DecoderOnlyDictPrompt,
|
prompt: DecoderOnlyDictPrompt,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> DecoderOnlyInputs: ...
|
) -> DecoderOnlyEngineInput: ...
|
||||||
|
|
||||||
def _prompt_to_llm_inputs(
|
def _prompt_to_llm_inputs(
|
||||||
self,
|
self,
|
||||||
prompt: SingletonDictPrompt,
|
prompt: SingletonDictPrompt,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> SingletonInputs:
|
) -> SingletonInput:
|
||||||
"""
|
|
||||||
Extract the singleton inputs from a prompt.
|
|
||||||
|
|
||||||
Arguments:
|
|
||||||
|
|
||||||
* prompt: single encoder or decoder input prompt
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
|
|
||||||
* [`SingletonInputs`][vllm.inputs.data.SingletonInputs] instance
|
|
||||||
"""
|
|
||||||
if "prompt_embeds" in prompt:
|
if "prompt_embeds" in prompt:
|
||||||
return self._process_embeds(prompt) # type: ignore[arg-type]
|
return self._process_embeds(prompt) # type: ignore[arg-type]
|
||||||
|
|
||||||
@@ -242,22 +231,7 @@ class InputPreprocessor:
|
|||||||
self,
|
self,
|
||||||
prompt: EncoderDecoderDictPrompt,
|
prompt: EncoderDecoderDictPrompt,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> EncoderDecoderInputs:
|
) -> EncoderDecoderInput:
|
||||||
"""
|
|
||||||
For encoder/decoder models only:
|
|
||||||
Process an input prompt into an
|
|
||||||
[`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
|
|
||||||
instance.
|
|
||||||
|
|
||||||
Arguments:
|
|
||||||
|
|
||||||
* prompt: an input prompt
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
|
|
||||||
* [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
|
|
||||||
instance
|
|
||||||
"""
|
|
||||||
encoder_prompt = prompt["encoder_prompt"]
|
encoder_prompt = prompt["encoder_prompt"]
|
||||||
decoder_prompt = prompt["decoder_prompt"]
|
decoder_prompt = prompt["decoder_prompt"]
|
||||||
|
|
||||||
@@ -270,12 +244,12 @@ class InputPreprocessor:
|
|||||||
self.renderer.mm_processor.skip_decoder_start_token
|
self.renderer.mm_processor.skip_decoder_start_token
|
||||||
)
|
)
|
||||||
|
|
||||||
return build_enc_dec_inputs(
|
return build_enc_dec_input(
|
||||||
encoder_inputs=self._prompt_to_llm_inputs(
|
encoder_input=self._prompt_to_llm_inputs(
|
||||||
encoder_prompt,
|
encoder_prompt,
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
tokenization_kwargs=tokenization_kwargs,
|
||||||
),
|
),
|
||||||
decoder_inputs=(
|
decoder_input=(
|
||||||
None
|
None
|
||||||
if decoder_prompt is None
|
if decoder_prompt is None
|
||||||
else self._prompt_to_llm_inputs(
|
else self._prompt_to_llm_inputs(
|
||||||
@@ -291,20 +265,7 @@ class InputPreprocessor:
|
|||||||
self,
|
self,
|
||||||
prompt: DecoderOnlyDictPrompt,
|
prompt: DecoderOnlyDictPrompt,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> DecoderOnlyInputs:
|
) -> DecoderOnlyEngineInput:
|
||||||
"""
|
|
||||||
For decoder-only models:
|
|
||||||
Process an input prompt into a
|
|
||||||
[`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance.
|
|
||||||
|
|
||||||
Arguments:
|
|
||||||
|
|
||||||
* prompt: input prompt
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
|
|
||||||
* [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance
|
|
||||||
"""
|
|
||||||
return self._prompt_to_llm_inputs(
|
return self._prompt_to_llm_inputs(
|
||||||
prompt,
|
prompt,
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
tokenization_kwargs=tokenization_kwargs,
|
||||||
@@ -314,7 +275,7 @@ class InputPreprocessor:
|
|||||||
self,
|
self,
|
||||||
prompt: PromptType,
|
prompt: PromptType,
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
tokenization_kwargs: dict[str, Any] | None = None,
|
||||||
) -> ProcessorInputs:
|
) -> EngineInput:
|
||||||
"""Preprocess the input prompt."""
|
"""Preprocess the input prompt."""
|
||||||
if self.model_config.is_encoder_decoder:
|
if self.model_config.is_encoder_decoder:
|
||||||
# Encoder-decoder model requires special mapping of
|
# Encoder-decoder model requires special mapping of
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from transformers.models.aria.processing_aria import AriaProcessor
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import get_tensor_model_parallel_rank
|
from vllm.distributed import get_tensor_model_parallel_rank
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.fused_moe import SharedFusedMoE
|
from vllm.model_executor.layers.fused_moe import SharedFusedMoE
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||||
@@ -24,7 +25,6 @@ from vllm.model_executor.model_loader.weight_utils import (
|
|||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -31,17 +31,16 @@ from transformers.models.qwen2_audio import Qwen2AudioEncoder
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import ModalityData, MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
from vllm.multimodal.parse import (
|
from vllm.multimodal.parse import (
|
||||||
DictEmbeddingItems,
|
DictEmbeddingItems,
|
||||||
ModalityData,
|
|
||||||
ModalityDataItems,
|
ModalityDataItems,
|
||||||
MultiModalDataItems,
|
MultiModalDataItems,
|
||||||
MultiModalDataParser,
|
MultiModalDataParser,
|
||||||
|
|||||||
@@ -17,9 +17,9 @@ from transformers.models.got_ocr2.image_processing_got_ocr2 import (
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import torch.nn as nn
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
@@ -24,7 +25,6 @@ from vllm.model_executor.layers.linear import (
|
|||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -9,8 +9,8 @@ from transformers.activations import GELUActivation
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import MultiModalDataDict
|
|
||||||
|
|
||||||
from .llava_next import (
|
from .llava_next import (
|
||||||
LlavaDummyInputsBuilder,
|
LlavaDummyInputsBuilder,
|
||||||
|
|||||||
@@ -15,11 +15,11 @@ from transformers import (
|
|||||||
|
|
||||||
from vllm.config import CacheConfig, VllmConfig
|
from vllm.config import CacheConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ from transformers import (
|
|||||||
from vllm.config import CacheConfig, VllmConfig
|
from vllm.config import CacheConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
@@ -43,7 +44,6 @@ from vllm.model_executor.model_loader.weight_utils import (
|
|||||||
from vllm.model_executor.utils import set_weight_attrs
|
from vllm.model_executor.utils import set_weight_attrs
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from transformers import (
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||||
|
from vllm.inputs import MultiModalDataDict, MultiModalInput
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.attention import Attention, MMEncoderAttention
|
from vllm.model_executor.layers.attention import Attention, MMEncoderAttention
|
||||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||||
@@ -32,9 +33,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
|||||||
from vllm.model_executor.models.interfaces import SupportsQuant
|
from vllm.model_executor.models.interfaces import SupportsQuant
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalInputs,
|
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
from vllm.multimodal.parse import (
|
from vllm.multimodal.parse import (
|
||||||
@@ -207,7 +206,7 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
|
|||||||
self,
|
self,
|
||||||
inputs: ProcessorInputs,
|
inputs: ProcessorInputs,
|
||||||
timing_ctx: TimingContext,
|
timing_ctx: TimingContext,
|
||||||
) -> MultiModalInputs:
|
) -> MultiModalInput:
|
||||||
if inputs.mm_data_items:
|
if inputs.mm_data_items:
|
||||||
if isinstance(inputs.prompt, str):
|
if isinstance(inputs.prompt, str):
|
||||||
if len(inputs.prompt) > 0:
|
if len(inputs.prompt) > 0:
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ from transformers.models.cohere2_vision.processing_cohere2_vision import (
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import MulAndSilu
|
from vllm.model_executor.layers.activation import MulAndSilu
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -28,7 +29,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
|||||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
import math
|
import math
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from typing import Literal, cast
|
from typing import Literal
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
@@ -14,7 +14,7 @@ from transformers import PretrainedConfig
|
|||||||
from vllm.compilation.decorators import support_torch_compile
|
from vllm.compilation.decorators import support_torch_compile
|
||||||
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
from vllm.inputs.data import PromptType
|
from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.attention import (
|
from vllm.model_executor.layers.attention import (
|
||||||
@@ -32,7 +32,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
|||||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
@@ -2047,14 +2046,11 @@ class CohereASRForConditionalGeneration(
|
|||||||
f"<|noitn|><|notimestamp|><|nodiarize|>"
|
f"<|noitn|><|notimestamp|><|nodiarize|>"
|
||||||
)
|
)
|
||||||
prompt_text = request_prompt if request_prompt else default_prompt
|
prompt_text = request_prompt if request_prompt else default_prompt
|
||||||
prompt = {
|
|
||||||
"prompt": prompt_text,
|
|
||||||
"multi_modal_data": {
|
|
||||||
"audio": (audio, stt_config.sample_rate),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
return cast(PromptType, prompt)
|
return TextPrompt(
|
||||||
|
prompt=prompt_text,
|
||||||
|
multi_modal_data={"audio": (audio, stt_config.sample_rate)},
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||||
|
|||||||
@@ -16,11 +16,11 @@ from transformers import BatchFeature
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
|
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
|
||||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from transformers import BatchFeature, CLIPVisionConfig
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.models.interfaces import (
|
from vllm.model_executor.models.interfaces import (
|
||||||
MultiModalEmbeddings,
|
MultiModalEmbeddings,
|
||||||
SupportsLoRA,
|
SupportsLoRA,
|
||||||
@@ -27,7 +28,6 @@ from vllm.model_executor.models.utils import (
|
|||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
NestedTensors,
|
NestedTensors,
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from transformers import BatchFeature
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.models.interfaces import (
|
from vllm.model_executor.models.interfaces import (
|
||||||
MultiModalEmbeddings,
|
MultiModalEmbeddings,
|
||||||
SupportsLoRA,
|
SupportsLoRA,
|
||||||
@@ -27,7 +28,6 @@ from vllm.model_executor.models.utils import (
|
|||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
NestedTensors,
|
NestedTensors,
|
||||||
|
|||||||
@@ -17,11 +17,11 @@ from transformers import BatchFeature
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.models.transformers.utils import replace_linear_class
|
from vllm.model_executor.models.transformers.utils import replace_linear_class
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from vllm.distributed.parallel_state import (
|
|||||||
get_tensor_model_parallel_rank,
|
get_tensor_model_parallel_rank,
|
||||||
get_tensor_model_parallel_world_size,
|
get_tensor_model_parallel_world_size,
|
||||||
)
|
)
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import (
|
from vllm.model_executor.layers.attention import (
|
||||||
MMEncoderAttention,
|
MMEncoderAttention,
|
||||||
@@ -54,7 +55,6 @@ from vllm.model_executor.models.utils import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.models.vision import get_vit_attn_backend
|
from vllm.model_executor.models.vision import get_vit_attn_backend
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import MultiModalDataDict
|
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.transformers_utils.configs.dotsocr import DotsOCRConfig, DotsVisionConfig
|
from vllm.transformers_utils.configs.dotsocr import DotsOCRConfig, DotsVisionConfig
|
||||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ from vllm.config import VllmConfig
|
|||||||
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||||
from vllm.distributed import parallel_state
|
from vllm.distributed import parallel_state
|
||||||
from vllm.distributed import utils as dist_utils
|
from vllm.distributed import utils as dist_utils
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import QuickGELU
|
from vllm.model_executor.layers.activation import QuickGELU
|
||||||
from vllm.model_executor.layers.attention import (
|
from vllm.model_executor.layers.attention import (
|
||||||
@@ -58,7 +59,6 @@ from vllm.model_executor.layers.rotary_embedding.common import (
|
|||||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFeatureSpec,
|
MultiModalFeatureSpec,
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from transformers import (
|
|||||||
|
|
||||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.inputs.data import PromptType
|
from vllm.inputs import MultiModalDataDict, PromptType
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
|
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
@@ -27,7 +27,6 @@ from vllm.model_executor.models.whisper_utils import (
|
|||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ from transformers import (
|
|||||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
from vllm.inputs.data import PromptType
|
from vllm.inputs import MultiModalDataDict, PromptType
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
|
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
|
||||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||||
@@ -37,7 +37,6 @@ from vllm.model_executor.models.whisper_utils import (
|
|||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -27,12 +27,12 @@ from transformers.modeling_outputs import BaseModelOutput
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -28,11 +28,11 @@ from transformers import BatchFeature, FuyuConfig, FuyuImageProcessor, FuyuProce
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||||
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
|
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -12,12 +12,12 @@ from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ from transformers.models.siglip import SiglipImageProcessorFast
|
|||||||
|
|
||||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.inputs.data import PromptType, TextPrompt
|
from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import RowParallelLinear
|
from vllm.model_executor.layers.linear import RowParallelLinear
|
||||||
@@ -32,7 +32,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
|||||||
from vllm.model_executor.models.whisper import ISO639_1_SUPPORTED_LANGS
|
from vllm.model_executor.models.whisper import ISO639_1_SUPPORTED_LANGS
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ from vllm.config import VllmConfig
|
|||||||
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state
|
from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state
|
||||||
from vllm.distributed import utils as dist_utils
|
from vllm.distributed import utils as dist_utils
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import (
|
from vllm.model_executor.layers.attention import (
|
||||||
MMEncoderAttention,
|
MMEncoderAttention,
|
||||||
@@ -74,7 +75,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
|||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFeatureSpec,
|
MultiModalFeatureSpec,
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from transformers import BatchFeature
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
|
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
|
||||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||||
@@ -32,7 +33,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
|||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFeatureSpec,
|
MultiModalFeatureSpec,
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
|
|||||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
|
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
|
||||||
from vllm.inputs.data import PromptType, TokensPrompt
|
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
@@ -27,13 +27,11 @@ from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
|||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
from vllm.multimodal.parse import (
|
from vllm.multimodal.parse import (
|
||||||
DictEmbeddingItems,
|
DictEmbeddingItems,
|
||||||
ModalityData,
|
|
||||||
ModalityDataItems,
|
ModalityDataItems,
|
||||||
MultiModalDataItems,
|
MultiModalDataItems,
|
||||||
MultiModalDataParser,
|
MultiModalDataParser,
|
||||||
|
|||||||
@@ -36,13 +36,12 @@ from transformers import BatchFeature, PretrainedConfig
|
|||||||
|
|
||||||
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.inputs.data import PromptType, TokensPrompt
|
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ from vllm.config import VllmConfig
|
|||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import parallel_state
|
from vllm.distributed import parallel_state
|
||||||
from vllm.distributed import utils as dist_utils
|
from vllm.distributed import utils as dist_utils
|
||||||
|
from vllm.inputs import ModalityData, MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||||
@@ -52,8 +53,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
|||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
ImageItem,
|
ImageItem,
|
||||||
ModalityData,
|
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFeatureSpec,
|
MultiModalFeatureSpec,
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
|
|||||||
@@ -18,10 +18,10 @@ from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ from transformers import BatchFeature
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.forward_context import set_forward_context
|
from vllm.forward_context import set_forward_context
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ from transformers import (
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
@@ -37,7 +38,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
|||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -30,8 +30,7 @@ from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
|||||||
from typing_extensions import Self, TypeIs
|
from typing_extensions import Self, TypeIs
|
||||||
|
|
||||||
from vllm.config import ModelConfig, SpeechToTextConfig
|
from vllm.config import ModelConfig, SpeechToTextConfig
|
||||||
from vllm.inputs import TokensPrompt
|
from vllm.inputs import PromptType, TokensPrompt
|
||||||
from vllm.inputs.data import PromptType
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
|
|||||||
@@ -23,12 +23,12 @@ from transformers.models.internvl.video_processing_internvl import (
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.models.interns1_vit import InternS1VisionModel
|
from vllm.model_executor.models.interns1_vit import InternS1VisionModel
|
||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from transformers import BatchFeature, PretrainedConfig
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||||
from vllm.model_executor.models.intern_vit import (
|
from vllm.model_executor.models.intern_vit import (
|
||||||
@@ -28,7 +29,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
|||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
BatchedTensorInputs,
|
BatchedTensorInputs,
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from vllm.config import ModelConfig, VllmConfig
|
|||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import parallel_state
|
from vllm.distributed import parallel_state
|
||||||
from vllm.distributed import utils as dist_utils
|
from vllm.distributed import utils as dist_utils
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
@@ -44,7 +45,6 @@ from vllm.model_executor.models.utils import (
|
|||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFeatureSpec,
|
MultiModalFeatureSpec,
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
|
|||||||
@@ -19,10 +19,10 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionCon
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ from transformers.utils import torch_int
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
|
from vllm.inputs import ModalityData, MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import (
|
from vllm.model_executor.layers.attention import (
|
||||||
MMEncoderAttention,
|
MMEncoderAttention,
|
||||||
@@ -41,8 +42,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
|||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
ImageItem,
|
ImageItem,
|
||||||
ModalityData,
|
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFeatureSpec,
|
MultiModalFeatureSpec,
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
|
|||||||
@@ -14,13 +14,13 @@ from transformers.activations import GELUActivation
|
|||||||
from transformers.feature_extraction_utils import BatchFeature
|
from transformers.feature_extraction_utils import BatchFeature
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
|
from vllm.inputs import ModalityData
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
ImageItem,
|
ImageItem,
|
||||||
ModalityData,
|
|
||||||
MultiModalFeatureSpec,
|
MultiModalFeatureSpec,
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from transformers import WhisperConfig as HFWhisperConfig
|
|||||||
|
|
||||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.inputs.data import PromptType, TokensPrompt
|
from vllm.inputs import PromptType, TokensPrompt
|
||||||
from vllm.model_executor.model_loader import DefaultModelLoader
|
from vllm.model_executor.model_loader import DefaultModelLoader
|
||||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||||
from vllm.model_executor.models.interfaces import (
|
from vllm.model_executor.models.interfaces import (
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from transformers import BatchFeature
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.layers.quantization.compressed_tensors import (
|
from vllm.model_executor.layers.quantization.compressed_tensors import (
|
||||||
@@ -35,7 +36,6 @@ from vllm.model_executor.models.kimi_k25_vit import (
|
|||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
NestedTensors,
|
NestedTensors,
|
||||||
|
|||||||
@@ -54,12 +54,12 @@ from transformers.activations import GELUActivation
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||||
from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP
|
from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP
|
||||||
from vllm.model_executor.models.moonvit import MoonVitPretrainedModel
|
from vllm.model_executor.models.moonvit import MoonVitPretrainedModel
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
NestedTensors,
|
NestedTensors,
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import (
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.forward_context import set_forward_context
|
from vllm.forward_context import set_forward_context
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||||
MambaStateCopyFunc,
|
MambaStateCopyFunc,
|
||||||
MambaStateCopyFuncCalculator,
|
MambaStateCopyFuncCalculator,
|
||||||
@@ -30,7 +31,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
|||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -20,17 +20,15 @@ from transformers.models.pixtral import PixtralProcessor
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalInputs,
|
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
mm_inputs,
|
|
||||||
)
|
)
|
||||||
from vllm.multimodal.parse import (
|
from vllm.multimodal.parse import (
|
||||||
ImageEmbeddingItems,
|
ImageEmbeddingItems,
|
||||||
@@ -777,7 +775,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
|
|||||||
self,
|
self,
|
||||||
inputs: ProcessorInputs,
|
inputs: ProcessorInputs,
|
||||||
timing_ctx: TimingContext,
|
timing_ctx: TimingContext,
|
||||||
) -> MultiModalInputs:
|
) -> MultiModalInput:
|
||||||
hf_config = self.info.get_hf_config()
|
hf_config = self.info.get_hf_config()
|
||||||
image_token_id = hf_config.image_token_index
|
image_token_id = hf_config.image_token_index
|
||||||
|
|
||||||
@@ -833,7 +831,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
|
|||||||
for modality, placeholders in mm_placeholders.items()
|
for modality, placeholders in mm_placeholders.items()
|
||||||
}
|
}
|
||||||
|
|
||||||
return mm_inputs(
|
return mm_input(
|
||||||
prompt_token_ids=prompt_ids,
|
prompt_token_ids=prompt_ids,
|
||||||
mm_kwargs=mm_kwargs,
|
mm_kwargs=mm_kwargs,
|
||||||
mm_hashes=mm_hashes,
|
mm_hashes=mm_hashes,
|
||||||
|
|||||||
@@ -11,11 +11,11 @@ from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProce
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.models.clip import CLIPVisionModel
|
from vllm.model_executor.models.clip import CLIPVisionModel
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -15,10 +15,10 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ from transformers import BatchFeature
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
@@ -48,7 +49,6 @@ from vllm.model_executor.layers.linear import (
|
|||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -41,9 +41,9 @@ from transformers.models.whisper.modeling_whisper import (
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import ModalityData, MultiModalDataDict
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
|
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
NestedTensors,
|
NestedTensors,
|
||||||
)
|
)
|
||||||
@@ -51,7 +51,6 @@ from vllm.multimodal.parse import (
|
|||||||
AudioItem,
|
AudioItem,
|
||||||
AudioProcessorItems,
|
AudioProcessorItems,
|
||||||
DictEmbeddingItems,
|
DictEmbeddingItems,
|
||||||
ModalityData,
|
|
||||||
ModalityDataItems,
|
ModalityDataItems,
|
||||||
MultiModalDataItems,
|
MultiModalDataItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ from typing_extensions import TypeVar
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import ModalityData, MultiModalDataDict
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.layers.resampler import (
|
from vllm.model_executor.layers.resampler import (
|
||||||
BaseResampler,
|
BaseResampler,
|
||||||
@@ -54,7 +55,6 @@ from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
|
|||||||
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
|
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
NestedTensors,
|
NestedTensors,
|
||||||
@@ -64,7 +64,6 @@ from vllm.multimodal.parse import (
|
|||||||
ImageItem,
|
ImageItem,
|
||||||
ImageProcessorItems,
|
ImageProcessorItems,
|
||||||
ImageSize,
|
ImageSize,
|
||||||
ModalityData,
|
|
||||||
ModalityDataItems,
|
ModalityDataItems,
|
||||||
MultiModalDataItems,
|
MultiModalDataItems,
|
||||||
MultiModalDataParser,
|
MultiModalDataParser,
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from transformers.models.pixtral import PixtralProcessor
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||||
@@ -18,7 +19,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
|||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ from vllm.compilation.decorators import (
|
|||||||
from vllm.config import VllmConfig, set_current_vllm_config
|
from vllm.config import VllmConfig, set_current_vllm_config
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
@@ -53,7 +54,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
|||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ from vllm.distributed import (
|
|||||||
split_tensor_along_last_dim,
|
split_tensor_along_last_dim,
|
||||||
tensor_model_parallel_all_gather,
|
tensor_model_parallel_all_gather,
|
||||||
)
|
)
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import MulAndSilu, QuickGELU, SiluAndMul
|
from vllm.model_executor.layers.activation import MulAndSilu, QuickGELU, SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention, MMEncoderAttention
|
from vllm.model_executor.layers.attention import Attention, MMEncoderAttention
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
@@ -49,7 +50,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
|||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ from vllm.distributed import (
|
|||||||
split_tensor_along_last_dim,
|
split_tensor_along_last_dim,
|
||||||
tensor_model_parallel_all_gather,
|
tensor_model_parallel_all_gather,
|
||||||
)
|
)
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import MulAndSilu, SiluAndMul, get_act_fn
|
from vllm.model_executor.layers.activation import MulAndSilu, SiluAndMul, get_act_fn
|
||||||
from vllm.model_executor.layers.attention import Attention, MMEncoderAttention
|
from vllm.model_executor.layers.attention import Attention, MMEncoderAttention
|
||||||
@@ -54,7 +55,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
|||||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
VideoItem,
|
VideoItem,
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from transformers import BatchFeature
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict, MultiModalInput
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import ReLUSquaredActivation
|
from vllm.model_executor.layers.activation import ReLUSquaredActivation
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
@@ -48,9 +49,7 @@ from vllm.multimodal.evs import (
|
|||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
AudioItem,
|
AudioItem,
|
||||||
BatchedTensorInputs,
|
BatchedTensorInputs,
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalInputs,
|
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
VideoItem,
|
VideoItem,
|
||||||
)
|
)
|
||||||
@@ -576,7 +575,7 @@ class NanoNemotronVLMultiModalProcessor(
|
|||||||
self,
|
self,
|
||||||
inputs: ProcessorInputs,
|
inputs: ProcessorInputs,
|
||||||
timing_ctx: TimingContext,
|
timing_ctx: TimingContext,
|
||||||
) -> MultiModalInputs:
|
) -> MultiModalInput:
|
||||||
use_audio_in_video = bool(
|
use_audio_in_video = bool(
|
||||||
inputs.hf_processor_mm_kwargs.get("use_audio_in_video", False)
|
inputs.hf_processor_mm_kwargs.get("use_audio_in_video", False)
|
||||||
)
|
)
|
||||||
@@ -632,7 +631,7 @@ class NanoNemotronVLMultiModalProcessor(
|
|||||||
for modality, placeholders in mm_placeholders.items()
|
for modality, placeholders in mm_placeholders.items()
|
||||||
}
|
}
|
||||||
|
|
||||||
return MultiModalInputs(
|
return MultiModalInput(
|
||||||
type="multimodal",
|
type="multimodal",
|
||||||
prompt_token_ids=prompt_ids,
|
prompt_token_ids=prompt_ids,
|
||||||
mm_kwargs=mm_info.kwargs,
|
mm_kwargs=mm_info.kwargs,
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from transformers import (
|
|||||||
from vllm.config import CacheConfig, VllmConfig
|
from vllm.config import CacheConfig, VllmConfig
|
||||||
from vllm.config.lora import LoRAConfig
|
from vllm.config.lora import LoRAConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||||
@@ -41,7 +42,6 @@ from vllm.model_executor.models.radio import RadioModel
|
|||||||
from vllm.model_executor.models.whisper import WhisperAttention, WhisperCrossAttention
|
from vllm.model_executor.models.whisper import WhisperAttention, WhisperCrossAttention
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import (
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -14,12 +14,10 @@ import torch.nn as nn
|
|||||||
from transformers import PretrainedConfig
|
from transformers import PretrainedConfig
|
||||||
|
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import (
|
from vllm.multimodal.inputs import BatchedTensorInputs
|
||||||
BatchedTensorInputs,
|
|
||||||
MultiModalDataDict,
|
|
||||||
)
|
|
||||||
from vllm.multimodal.parse import (
|
from vllm.multimodal.parse import (
|
||||||
ImageEmbeddingItems,
|
ImageEmbeddingItems,
|
||||||
ImageProcessorItems,
|
ImageProcessorItems,
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user