Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+160 -125
View File
@@ -14,25 +14,37 @@ from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.logger import RequestLogger
# yapf conflicts with isort for this docstring
# yapf: disable
from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
EmbeddingCompletionRequest,
EmbeddingRequest,
EmbeddingResponse,
EmbeddingResponseData,
ErrorResponse, UsageInfo)
from vllm.entrypoints.openai.serving_engine import (EmbeddingServeContext,
OpenAIServing,
ServeContext,
TextTokensPrompt)
from vllm.entrypoints.openai.protocol import (
EmbeddingChatRequest,
EmbeddingCompletionRequest,
EmbeddingRequest,
EmbeddingResponse,
EmbeddingResponseData,
ErrorResponse,
UsageInfo,
)
from vllm.entrypoints.openai.serving_engine import (
EmbeddingServeContext,
OpenAIServing,
ServeContext,
TextTokensPrompt,
)
# yapf: enable
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.renderer import RenderConfig
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
from vllm.logger import init_logger
from vllm.outputs import (EmbeddingOutput, EmbeddingRequestOutput,
PoolingOutput, PoolingRequestOutput, RequestOutput)
from vllm.outputs import (
EmbeddingOutput,
EmbeddingRequestOutput,
PoolingOutput,
PoolingRequestOutput,
RequestOutput,
)
from vllm.pooling_params import PoolingParams
from vllm.utils import chunk_list
@@ -55,7 +67,6 @@ def _get_embedding(
class EmbeddingMixin(OpenAIServing):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -63,9 +74,13 @@ class EmbeddingMixin(OpenAIServing):
# Avoid repeated attribute lookups
self.supports_chunked_processing = bool(
pooler_config and pooler_config.enable_chunked_processing)
self.max_embed_len = (pooler_config.max_embed_len if pooler_config
and pooler_config.max_embed_len else None)
pooler_config and pooler_config.enable_chunked_processing
)
self.max_embed_len = (
pooler_config.max_embed_len
if pooler_config and pooler_config.max_embed_len
else None
)
@override
async def _preprocess(
@@ -88,10 +103,8 @@ class EmbeddingMixin(OpenAIServing):
ctx.request,
tokenizer,
ctx.request.messages,
chat_template=ctx.request.chat_template
or ctx.chat_template,
chat_template_content_format=ctx.
chat_template_content_format,
chat_template=ctx.request.chat_template or ctx.chat_template,
chat_template_content_format=ctx.chat_template_content_format,
add_generation_prompt=ctx.request.add_generation_prompt,
continue_final_message=False,
add_special_tokens=ctx.request.add_special_tokens,
@@ -106,8 +119,7 @@ class EmbeddingMixin(OpenAIServing):
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(str(e))
def _build_render_config(
self, request: EmbeddingCompletionRequest) -> RenderConfig:
def _build_render_config(self, request: EmbeddingCompletionRequest) -> RenderConfig:
# Set max_length based on chunked processing capability
if self._should_use_chunked_processing(request):
max_length = None
@@ -117,7 +129,8 @@ class EmbeddingMixin(OpenAIServing):
return RenderConfig(
max_length=max_length,
truncate_prompt_tokens=request.truncate_prompt_tokens,
add_special_tokens=request.add_special_tokens)
add_special_tokens=request.add_special_tokens,
)
@override
def _build_response(
@@ -127,16 +140,16 @@ class EmbeddingMixin(OpenAIServing):
items: list[EmbeddingResponseData] = []
num_prompt_tokens = 0
final_res_batch_checked = cast(list[PoolingRequestOutput],
ctx.final_res_batch)
final_res_batch_checked = cast(list[PoolingRequestOutput], ctx.final_res_batch)
for idx, final_res in enumerate(final_res_batch_checked):
embedding_res = EmbeddingRequestOutput.from_base(final_res)
item = EmbeddingResponseData(
index=idx,
embedding=_get_embedding(embedding_res.outputs,
ctx.request.encoding_format),
embedding=_get_embedding(
embedding_res.outputs, ctx.request.encoding_format
),
)
prompt_token_ids = final_res.prompt_token_ids
@@ -162,10 +175,10 @@ class EmbeddingMixin(OpenAIServing):
def _should_use_chunked_processing(self, request) -> bool:
"""Check if chunked processing should be used for this request."""
return isinstance(
request,
(EmbeddingCompletionRequest,
EmbeddingChatRequest)) and self.supports_chunked_processing
return (
isinstance(request, (EmbeddingCompletionRequest, EmbeddingChatRequest))
and self.supports_chunked_processing
)
async def _process_chunked_request(
self,
@@ -183,25 +196,27 @@ class EmbeddingMixin(OpenAIServing):
max_pos_embeddings = self._get_max_position_embeddings()
# Process all chunks for MEAN aggregation
for chunk_idx, chunk_tokens in enumerate(
chunk_list(token_ids, max_pos_embeddings)):
chunk_list(token_ids, max_pos_embeddings)
):
# Create a request ID for this chunk
chunk_request_id = (f"{ctx.request_id}-prompt-{prompt_idx}-"
f"chunk-{chunk_idx}")
chunk_request_id = f"{ctx.request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
# Create engine prompt for this chunk
chunk_engine_prompt = EngineTokensPrompt(
prompt_token_ids=chunk_tokens)
chunk_engine_prompt = EngineTokensPrompt(prompt_token_ids=chunk_tokens)
# Create chunk request prompt for logging
chunk_text = ""
chunk_request_prompt = TextTokensPrompt(
prompt=chunk_text, prompt_token_ids=chunk_tokens)
prompt=chunk_text, prompt_token_ids=chunk_tokens
)
# Log the chunk
self._log_inputs(chunk_request_id,
chunk_request_prompt,
params=pooling_params,
lora_request=ctx.lora_request)
self._log_inputs(
chunk_request_id,
chunk_request_prompt,
params=pooling_params,
lora_request=ctx.lora_request,
)
# Create generator for this chunk and wrap it to return indices
original_generator = self.engine_client.encode(
@@ -227,8 +242,7 @@ class EmbeddingMixin(OpenAIServing):
token_num = len(input_ids)
# Note: EmbeddingRequest doesn't have max_tokens
if isinstance(request,
(EmbeddingCompletionRequest, EmbeddingChatRequest)):
if isinstance(request, (EmbeddingCompletionRequest, EmbeddingChatRequest)):
# Check if chunked processing is enabled for pooling models
enable_chunked = self._should_use_chunked_processing(request)
@@ -248,13 +262,15 @@ class EmbeddingMixin(OpenAIServing):
validation_error_msg = (
"This model's {length_type} is {max_length_value} tokens. "
"However, you requested {token_num} tokens in the input for "
"embedding generation. Please reduce the length of the input.")
"embedding generation. Please reduce the length of the input."
)
chunked_processing_error_msg = (
"This model's {length_type} is {max_length_value} tokens. "
"However, you requested {token_num} tokens in the input for "
"embedding generation. Please reduce the length of the input "
"or enable chunked processing.")
"or enable chunked processing."
)
# Check if input exceeds max length
if token_num > max_length_value:
@@ -262,7 +278,9 @@ class EmbeddingMixin(OpenAIServing):
validation_error_msg.format(
length_type=length_type,
max_length_value=max_length_value,
token_num=token_num))
token_num=token_num,
)
)
# Check for chunked processing
# when exceeding max_position_embeddings
@@ -271,25 +289,31 @@ class EmbeddingMixin(OpenAIServing):
# Allow long inputs when chunked processing is enabled
logger.info(
"Input length %s exceeds max_position_embeddings "
"%s, will use chunked processing", token_num,
max_pos_embeddings)
"%s, will use chunked processing",
token_num,
max_pos_embeddings,
)
else:
raise ValueError(
chunked_processing_error_msg.format(
length_type="maximum position embeddings length",
max_length_value=max_pos_embeddings,
token_num=token_num))
token_num=token_num,
)
)
return TextTokensPrompt(prompt=input_text,
prompt_token_ids=input_ids)
return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
# For other request types, use the parent's implementation
return super()._validate_input(request, input_ids, input_text)
def _is_text_tokens_prompt(self, prompt) -> bool:
"""Check if a prompt is a TextTokensPrompt (has prompt_token_ids)."""
return (isinstance(prompt, dict) and "prompt_token_ids" in prompt
and "prompt_embeds" not in prompt)
return (
isinstance(prompt, dict)
and "prompt_token_ids" in prompt
and "prompt_embeds" not in prompt
)
async def _create_single_prompt_generator(
self,
@@ -302,10 +326,12 @@ class EmbeddingMixin(OpenAIServing):
"""Create a generator for a single prompt using standard processing."""
request_id_item = f"{ctx.request_id}-{prompt_index}"
self._log_inputs(request_id_item,
engine_prompt,
params=pooling_params,
lora_request=ctx.lora_request)
self._log_inputs(
request_id_item,
engine_prompt,
params=pooling_params,
lora_request=ctx.lora_request,
)
# Return the original generator without wrapping
return self.engine_client.encode(
@@ -333,13 +359,16 @@ class EmbeddingMixin(OpenAIServing):
return await super()._prepare_generators(ctx)
# Custom logic for chunked processing
generators: list[AsyncGenerator[Union[RequestOutput,
PoolingRequestOutput],
None]] = []
generators: list[
AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]
] = []
try:
trace_headers = (None if ctx.raw_request is None else await
self._get_trace_headers(ctx.raw_request.headers))
trace_headers = (
None
if ctx.raw_request is None
else await self._get_trace_headers(ctx.raw_request.headers)
)
pooling_params = self._create_pooling_params(ctx)
if isinstance(pooling_params, ErrorResponse):
@@ -352,8 +381,7 @@ class EmbeddingMixin(OpenAIServing):
return self.create_error_response(str(e))
if ctx.engine_prompts is None:
return self.create_error_response(
"Engine prompts not available")
return self.create_error_response("Engine prompts not available")
max_pos_embeddings = self._get_max_position_embeddings()
@@ -363,21 +391,22 @@ class EmbeddingMixin(OpenAIServing):
# Cast to TextTokensPrompt since we've verified
# prompt_token_ids
text_tokens_prompt = cast(TextTokensPrompt, engine_prompt)
if (len(text_tokens_prompt["prompt_token_ids"])
> max_pos_embeddings):
if len(text_tokens_prompt["prompt_token_ids"]) > max_pos_embeddings:
# Use chunked processing for this prompt
chunk_generators = await self._process_chunked_request(
ctx, text_tokens_prompt, pooling_params,
trace_headers, i)
ctx, text_tokens_prompt, pooling_params, trace_headers, i
)
generators.extend(chunk_generators)
continue
# Normal processing for short prompts or non-token prompts
generator = await self._create_single_prompt_generator(
ctx, engine_prompt, pooling_params, trace_headers, i)
ctx, engine_prompt, pooling_params, trace_headers, i
)
generators.append(generator)
from vllm.utils import merge_async_iterators
ctx.result_generator = merge_async_iterators(*generators)
return None
@@ -401,8 +430,7 @@ class EmbeddingMixin(OpenAIServing):
ctx = cast(EmbeddingServeContext, ctx)
try:
if ctx.engine_prompts is None:
return self.create_error_response(
"Engine prompts not available")
return self.create_error_response("Engine prompts not available")
# Check if we used chunked processing
use_chunked = self._should_use_chunked_processing(ctx.request)
@@ -411,8 +439,7 @@ class EmbeddingMixin(OpenAIServing):
return await super()._collect_batch(ctx=ctx)
if ctx.result_generator is None:
return self.create_error_response(
"Result generator not available")
return self.create_error_response("Result generator not available")
# Online aggregation for chunked requests to
# minimize memory usage
@@ -433,10 +460,10 @@ class EmbeddingMixin(OpenAIServing):
# Initialize aggregator for this prompt if needed
if prompt_idx not in prompt_aggregators:
prompt_aggregators[prompt_idx] = {
'weighted_sum': None,
'total_weight': 0,
'chunk_count': 0,
'request_id': result.request_id.split("-chunk-")[0]
"weighted_sum": None,
"total_weight": 0,
"chunk_count": 0,
"request_id": result.request_id.split("-chunk-")[0],
}
aggregator = prompt_aggregators[prompt_idx]
@@ -448,44 +475,45 @@ class EmbeddingMixin(OpenAIServing):
return self.create_error_response(
f"Expected PoolingRequestOutput for "
f"chunked embedding, got "
f"{type(result).__name__}")
f"{type(result).__name__}"
)
# Handle both PoolingOutput and
# EmbeddingOutput types
if hasattr(result.outputs, 'data'):
if hasattr(result.outputs, "data"):
# PoolingOutput case
embedding_data = result.outputs.data
elif hasattr(result.outputs, 'embedding'):
elif hasattr(result.outputs, "embedding"):
# EmbeddingOutput case -
# convert embedding list to tensor
embedding_data = result.outputs.embedding
else:
return self.create_error_response(
f"Unsupported output type: "
f"{type(result.outputs).__name__}")
f"Unsupported output type: {type(result.outputs).__name__}"
)
if not isinstance(embedding_data, torch.Tensor):
embedding_data = torch.tensor(embedding_data,
dtype=torch.float32)
embedding_data = torch.tensor(
embedding_data, dtype=torch.float32
)
if result.prompt_token_ids is None:
return self.create_error_response(
"prompt_token_ids cannot be None for "
"chunked processing")
"prompt_token_ids cannot be None for chunked processing"
)
weight = len(result.prompt_token_ids)
weighted_embedding = embedding_data.to(
dtype=torch.float32) * weight
weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
if aggregator['weighted_sum'] is None:
if aggregator["weighted_sum"] is None:
# First chunk
aggregator['weighted_sum'] = weighted_embedding
aggregator["weighted_sum"] = weighted_embedding
else:
# Accumulate
aggregator['weighted_sum'] += weighted_embedding
aggregator["weighted_sum"] += weighted_embedding
aggregator['total_weight'] += weight
aggregator['chunk_count'] += 1
aggregator["total_weight"] += weight
aggregator["chunk_count"] += 1
else:
# Non-chunked result - extract prompt_idx from request_id
parts = result.request_id.split("-")
@@ -496,11 +524,13 @@ class EmbeddingMixin(OpenAIServing):
prompt_idx = result_idx # Fallback to result_idx
short_prompts_results[prompt_idx] = cast(
PoolingRequestOutput, result)
PoolingRequestOutput, result
)
# Finalize aggregated results
final_res_batch: list[Union[PoolingRequestOutput,
EmbeddingRequestOutput]] = []
final_res_batch: list[
Union[PoolingRequestOutput, EmbeddingRequestOutput]
] = []
num_prompts = len(ctx.engine_prompts)
for prompt_idx in range(num_prompts):
@@ -508,55 +538,57 @@ class EmbeddingMixin(OpenAIServing):
# Finalize MEAN aggregation for this chunked prompt
aggregator = prompt_aggregators[prompt_idx]
weighted_sum = aggregator['weighted_sum']
total_weight = aggregator['total_weight']
if (weighted_sum is not None
and isinstance(weighted_sum, torch.Tensor)
and isinstance(total_weight,
(int, float)) and total_weight > 0):
weighted_sum = aggregator["weighted_sum"]
total_weight = aggregator["total_weight"]
if (
weighted_sum is not None
and isinstance(weighted_sum, torch.Tensor)
and isinstance(total_weight, (int, float))
and total_weight > 0
):
# Compute final mean embedding
final_embedding = weighted_sum / total_weight
# Create a PoolingRequestOutput
# for the aggregated result
pooling_output_data = PoolingOutput(
data=final_embedding)
pooling_output_data = PoolingOutput(data=final_embedding)
# Get original prompt token IDs for this prompt
original_prompt = ctx.engine_prompts[prompt_idx]
if not self._is_text_tokens_prompt(original_prompt):
return self.create_error_response(
f"Chunked prompt {prompt_idx} is not a "
f"TextTokensPrompt")
f"Chunked prompt {prompt_idx} is not a TextTokensPrompt"
)
original_token_ids = cast(
TextTokensPrompt,
original_prompt)["prompt_token_ids"]
original_token_ids = cast(TextTokensPrompt, original_prompt)[
"prompt_token_ids"
]
pooling_request_output = PoolingRequestOutput(
request_id=aggregator['request_id'],
request_id=aggregator["request_id"],
prompt_token_ids=original_token_ids,
outputs=pooling_output_data,
finished=True)
finished=True,
)
final_res_batch.append(pooling_request_output)
else:
return self.create_error_response(
f"Failed to aggregate chunks "
f"for prompt {prompt_idx}")
f"Failed to aggregate chunks for prompt {prompt_idx}"
)
elif prompt_idx in short_prompts_results:
final_res_batch.append(
cast(PoolingRequestOutput,
short_prompts_results[prompt_idx]))
cast(PoolingRequestOutput, short_prompts_results[prompt_idx])
)
else:
return self.create_error_response(
f"Result not found for prompt {prompt_idx}")
f"Result not found for prompt {prompt_idx}"
)
ctx.final_res_batch = cast(
list[Union[RequestOutput, PoolingRequestOutput]],
final_res_batch)
list[Union[RequestOutput, PoolingRequestOutput]], final_res_batch
)
return None
@@ -579,11 +611,13 @@ class OpenAIServingEmbedding(EmbeddingMixin):
trust_request_chat_template: bool = False,
log_error_stack: bool = False,
) -> None:
super().__init__(engine_client=engine_client,
model_config=model_config,
models=models,
request_logger=request_logger,
log_error_stack=log_error_stack)
super().__init__(
engine_client=engine_client,
model_config=model_config,
models=models,
request_logger=request_logger,
log_error_stack=log_error_stack,
)
self.chat_template = chat_template
self.chat_template_content_format: Final = chat_template_content_format
@@ -603,7 +637,8 @@ class OpenAIServingEmbedding(EmbeddingMixin):
model_name = self.models.model_name()
request_id = (
f"{self.request_id_prefix}-"
f"{self._base_request_id(raw_request, request.request_id)}")
f"{self._base_request_id(raw_request, request.request_id)}"
)
ctx = EmbeddingServeContext(
request=request,