[CI/Build] mypy: Resolve some errors from checking vllm/engine (#9267)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
@@ -3,7 +3,7 @@ import dataclasses
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
|
||||
Tuple, Type, Union)
|
||||
Tuple, Type, Union, cast)
|
||||
|
||||
import torch
|
||||
|
||||
@@ -89,7 +89,7 @@ class EngineArgs:
|
||||
trust_remote_code: bool = False
|
||||
download_dir: Optional[str] = None
|
||||
load_format: str = 'auto'
|
||||
config_format: str = 'auto'
|
||||
config_format: ConfigFormat = ConfigFormat.AUTO
|
||||
dtype: str = 'auto'
|
||||
kv_cache_dtype: str = 'auto'
|
||||
quantization_param_path: Optional[str] = None
|
||||
@@ -181,7 +181,7 @@ class EngineArgs:
|
||||
scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
|
||||
|
||||
def __post_init__(self):
|
||||
if self.tokenizer is None:
|
||||
if not self.tokenizer:
|
||||
self.tokenizer = self.model
|
||||
|
||||
# Setup plugins
|
||||
@@ -837,7 +837,8 @@ class EngineArgs:
|
||||
def create_model_config(self) -> ModelConfig:
|
||||
return ModelConfig(
|
||||
model=self.model,
|
||||
tokenizer=self.tokenizer,
|
||||
# We know this is not None because we set it in __post_init__
|
||||
tokenizer=cast(str, self.tokenizer),
|
||||
tokenizer_mode=self.tokenizer_mode,
|
||||
trust_remote_code=self.trust_remote_code,
|
||||
dtype=self.dtype,
|
||||
@@ -908,8 +909,9 @@ class EngineArgs:
|
||||
self.enable_prefix_caching = False
|
||||
|
||||
cache_config = CacheConfig(
|
||||
# neuron needs block_size = max_model_len
|
||||
block_size=self.block_size if self.device != "neuron" else
|
||||
self.max_model_len, # neuron needs block_size = max_model_len
|
||||
(self.max_model_len if self.max_model_len is not None else 0),
|
||||
gpu_memory_utilization=self.gpu_memory_utilization,
|
||||
swap_space=self.swap_space,
|
||||
cache_dtype=self.kv_cache_dtype,
|
||||
|
||||
@@ -6,7 +6,7 @@ from functools import partial
|
||||
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
|
||||
Iterable, List, Mapping, NamedTuple, Optional)
|
||||
from typing import Sequence as GenericSequence
|
||||
from typing import Set, Type, Union, overload
|
||||
from typing import Set, Type, Union, cast, overload
|
||||
|
||||
import torch
|
||||
from typing_extensions import TypeVar
|
||||
@@ -44,7 +44,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
|
||||
Sequence, SequenceGroup, SequenceGroupMetadata,
|
||||
SequenceStatus)
|
||||
SequenceGroupOutput, SequenceStatus)
|
||||
from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
|
||||
init_tracer)
|
||||
from vllm.transformers_utils.config import try_get_generation_config
|
||||
@@ -188,7 +188,7 @@ class LLMEngine:
|
||||
raise TypeError(f"Expected output of type {output_type}, "
|
||||
f"but found type {type(output)}")
|
||||
|
||||
return output
|
||||
return cast(_O, output)
|
||||
|
||||
@classmethod
|
||||
def validate_outputs(
|
||||
@@ -1039,6 +1039,7 @@ class LLMEngine:
|
||||
scheduler_outputs.scheduled_seq_groups)
|
||||
|
||||
has_multiple_outputs: bool = len(outputs) > 1
|
||||
outputs_by_sequence_group: List[List[SequenceGroupOutput]]
|
||||
if has_multiple_outputs:
|
||||
assert self.scheduler_config.is_multi_step or \
|
||||
self.speculative_config
|
||||
@@ -1084,6 +1085,7 @@ class LLMEngine:
|
||||
finished_before.append(i)
|
||||
continue
|
||||
|
||||
output: List[SequenceGroupOutput]
|
||||
if has_multiple_outputs:
|
||||
output = outputs_by_sequence_group[i]
|
||||
else:
|
||||
@@ -1096,7 +1098,7 @@ class LLMEngine:
|
||||
seq_group, seq_group_meta, is_first_step_output)
|
||||
else:
|
||||
seq_group.update_num_computed_tokens(
|
||||
seq_group_meta.token_chunk_size)
|
||||
seq_group_meta.token_chunk_size or 0)
|
||||
|
||||
if outputs:
|
||||
for o in outputs:
|
||||
@@ -1104,13 +1106,13 @@ class LLMEngine:
|
||||
and seq_group.metrics is not None):
|
||||
if seq_group.metrics.model_forward_time is not None:
|
||||
seq_group.metrics.model_forward_time += (
|
||||
o.model_forward_time)
|
||||
o.model_forward_time or 0)
|
||||
else:
|
||||
seq_group.metrics.model_forward_time = (
|
||||
o.model_forward_time)
|
||||
if seq_group.metrics.model_execute_time is not None:
|
||||
seq_group.metrics.model_execute_time += (
|
||||
o.model_execute_time)
|
||||
o.model_execute_time or 0)
|
||||
else:
|
||||
seq_group.metrics.model_execute_time = (
|
||||
o.model_execute_time)
|
||||
@@ -1236,8 +1238,10 @@ class LLMEngine:
|
||||
seq_group, seq_group_metadata,
|
||||
seq_group.state.num_steps == 1)
|
||||
else:
|
||||
seq_group.update_num_computed_tokens(
|
||||
seq_group_metadata.token_chunk_size)
|
||||
token_chunk_size = (seq_group_metadata.token_chunk_size
|
||||
if seq_group_metadata.token_chunk_size
|
||||
is not None else 0)
|
||||
seq_group.update_num_computed_tokens(token_chunk_size)
|
||||
|
||||
if seq_group_metadata.do_sample:
|
||||
assert len(sequence_group_outputs.samples) == 1, (
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Counter as CollectionsCounter
|
||||
from typing import Dict, List, Optional, Union
|
||||
from typing import Dict, List, Optional, Type, Union, cast
|
||||
|
||||
import numpy as np
|
||||
import prometheus_client
|
||||
@@ -249,10 +249,11 @@ class _RayHistogramWrapper:
|
||||
labelnames: Optional[List[str]] = None,
|
||||
buckets: Optional[List[float]] = None):
|
||||
labelnames_tuple = tuple(labelnames) if labelnames else None
|
||||
boundaries = buckets if buckets else []
|
||||
self._histogram = ray_metrics.Histogram(name=name,
|
||||
description=documentation,
|
||||
tag_keys=labelnames_tuple,
|
||||
boundaries=buckets)
|
||||
boundaries=boundaries)
|
||||
|
||||
def labels(self, **labels):
|
||||
self._histogram.set_default_tags(labels)
|
||||
@@ -267,9 +268,12 @@ class RayMetrics(Metrics):
|
||||
RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
|
||||
Provides the same metrics as Metrics but uses Ray's util.metrics library.
|
||||
"""
|
||||
_gauge_cls = _RayGaugeWrapper
|
||||
_counter_cls = _RayCounterWrapper
|
||||
_histogram_cls = _RayHistogramWrapper
|
||||
_gauge_cls: Type[prometheus_client.Gauge] = cast(
|
||||
Type[prometheus_client.Gauge], _RayGaugeWrapper)
|
||||
_counter_cls: Type[prometheus_client.Counter] = cast(
|
||||
Type[prometheus_client.Counter], _RayCounterWrapper)
|
||||
_histogram_cls: Type[prometheus_client.Histogram] = cast(
|
||||
Type[prometheus_client.Histogram], _RayHistogramWrapper)
|
||||
|
||||
def __init__(self, labelnames: List[str], max_model_len: int):
|
||||
if ray_metrics is None:
|
||||
|
||||
@@ -3,7 +3,7 @@ import copy
|
||||
import pickle
|
||||
from contextlib import contextmanager, suppress
|
||||
from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping,
|
||||
Optional, Union, overload)
|
||||
Optional, Union, cast, overload)
|
||||
|
||||
import cloudpickle
|
||||
import zmq
|
||||
@@ -513,9 +513,14 @@ class MQLLMEngineClient(EngineClient):
|
||||
assert (prompt is not None and pooling_params is not None
|
||||
and request_id is not None)
|
||||
|
||||
return self._process_request(prompt, pooling_params, request_id,
|
||||
lora_request, trace_headers, None,
|
||||
priority)
|
||||
return cast(
|
||||
AsyncGenerator[EmbeddingRequestOutput, None],
|
||||
self._process_request(prompt,
|
||||
pooling_params,
|
||||
request_id,
|
||||
lora_request,
|
||||
trace_headers,
|
||||
priority=priority))
|
||||
|
||||
async def _process_request(
|
||||
self,
|
||||
@@ -543,7 +548,9 @@ class MQLLMEngineClient(EngineClient):
|
||||
build_guided_decoding_logits_processor_async(
|
||||
sampling_params=params,
|
||||
tokenizer=await self.get_tokenizer(lora_request),
|
||||
default_guided_backend=self.decoding_config.guided_decoding_backend
|
||||
default_guided_backend=(self.decoding_config.guided_decoding_backend
|
||||
if self.decoding_config
|
||||
else DecodingConfig.guided_decoding_backend),
|
||||
)
|
||||
|
||||
# 1) Create output queue for this requests.
|
||||
|
||||
@@ -73,11 +73,9 @@ class MQLLMEngine:
|
||||
# For MQLLMEngine, we can use cached outputs, since each new request
|
||||
# output is immediately pickled and send over the socket, which frees
|
||||
# the python object to be reused again.
|
||||
use_cached_outputs = True
|
||||
kwargs['use_cached_outputs'] = True
|
||||
|
||||
self.engine = LLMEngine(*args,
|
||||
**kwargs,
|
||||
use_cached_outputs=use_cached_outputs)
|
||||
self.engine = LLMEngine(*args, **kwargs)
|
||||
self.log_requests = log_requests
|
||||
|
||||
self.use_async_sockets = use_async_sockets
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import functools
|
||||
from typing import Callable, List
|
||||
from typing import Callable, List, cast
|
||||
|
||||
from vllm.core.scheduler import Scheduler
|
||||
from vllm.engine.output_processor.interfaces import (
|
||||
@@ -9,8 +9,10 @@ from vllm.engine.output_processor.single_step import (
|
||||
from vllm.engine.output_processor.stop_checker import StopChecker
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Sequence, SequenceGroup,
|
||||
SequenceGroupOutput, SequenceOutput, SequenceStatus)
|
||||
from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
|
||||
CompletionSequenceGroupOutput, Sequence,
|
||||
SequenceGroup, SequenceGroupOutput, SequenceOutput,
|
||||
SequenceStatus)
|
||||
from vllm.transformers_utils.detokenizer import Detokenizer
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.utils import Counter
|
||||
@@ -57,6 +59,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
|
||||
"""
|
||||
for output in outputs:
|
||||
# Concatenate single-step prompt logprob processing results.
|
||||
assert isinstance(output, CompletionSequenceGroupOutput)
|
||||
single_step_process_prompt_logprob(self, seq_group, output)
|
||||
|
||||
@staticmethod
|
||||
@@ -100,8 +103,18 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
|
||||
"Beam search not supported in multi-step decoding.")
|
||||
seq = seqs[0]
|
||||
seq_id = seq.seq_id
|
||||
assert all(
|
||||
[seq_id == output.samples[0].parent_seq_id for output in outputs])
|
||||
# This method is defined in the more generic
|
||||
# SequenceGroupOutputProcessor, but here we assume that the outputs are
|
||||
# of a more specific type.
|
||||
assert all([
|
||||
isinstance(output, CompletionSequenceGroupOutput)
|
||||
for output in outputs
|
||||
])
|
||||
compl_outputs = cast(List[CompletionSequenceGroupOutput], outputs)
|
||||
assert all([
|
||||
seq_id == output.samples[0].parent_seq_id
|
||||
for output in compl_outputs
|
||||
])
|
||||
|
||||
if is_async:
|
||||
# Async case: We process tokens one by one. Here, we know the token
|
||||
@@ -113,7 +126,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
|
||||
|
||||
# Since there's only one sequence per sequence group,
|
||||
# we can take the first sample.
|
||||
samples = [output.samples[0] for output in outputs]
|
||||
samples = [output.samples[0] for output in compl_outputs]
|
||||
|
||||
# entries in sample tokens may be invalid (eg. due to spec decode
|
||||
# rejecting tokens).
|
||||
|
||||
@@ -6,8 +6,9 @@ from vllm.engine.output_processor.interfaces import (
|
||||
SequenceGroupOutputProcessor)
|
||||
from vllm.engine.output_processor.stop_checker import StopChecker
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
|
||||
SequenceOutput, SequenceStatus)
|
||||
from vllm.sequence import (CompletionSequenceGroupOutput, Sequence,
|
||||
SequenceGroup, SequenceGroupOutput, SequenceOutput,
|
||||
SequenceStatus)
|
||||
from vllm.transformers_utils.detokenizer import Detokenizer
|
||||
from vllm.utils import Counter
|
||||
|
||||
@@ -16,7 +17,7 @@ logger = init_logger(__name__)
|
||||
|
||||
def single_step_process_prompt_logprob(
|
||||
sg_output_proc: SequenceGroupOutputProcessor, seq_group: SequenceGroup,
|
||||
output: SequenceGroupOutput) -> None:
|
||||
output: CompletionSequenceGroupOutput) -> None:
|
||||
"""Process prompt logprobs associated with the :class:`SequenceGroupOutput`
|
||||
for a given step.
|
||||
|
||||
@@ -106,6 +107,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
|
||||
"""
|
||||
assert len(outputs) == 1, ("Single step should only has 1 output.")
|
||||
output = outputs[0]
|
||||
assert isinstance(output, CompletionSequenceGroupOutput)
|
||||
single_step_process_prompt_logprob(self, seq_group, output)
|
||||
|
||||
def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
|
||||
|
||||
@@ -57,7 +57,7 @@ class StopChecker:
|
||||
# Check if a stop token was encountered.
|
||||
# This assumes a single token produced per step.
|
||||
last_token_id = seq.get_last_token_id()
|
||||
if last_token_id in sampling_params.stop_token_ids:
|
||||
if last_token_id in (sampling_params.stop_token_ids or ()):
|
||||
if new_char_count and (
|
||||
not sampling_params.include_stop_str_in_output):
|
||||
# Remove last token
|
||||
@@ -92,7 +92,7 @@ class StopChecker:
|
||||
|
||||
Returns the stop string if matched or else None.
|
||||
"""
|
||||
if not new_char_count:
|
||||
if not new_char_count or not sampling_params.stop:
|
||||
return None
|
||||
|
||||
for stop_str in sampling_params.stop:
|
||||
|
||||
@@ -1,22 +1,25 @@
|
||||
from typing import List
|
||||
from typing import Sequence as GenericSequence
|
||||
from typing import Union
|
||||
from typing import cast
|
||||
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.sequence import PoolerOutput, SequenceGroupOutput
|
||||
from vllm.sequence import CompletionSequenceGroupOutput, SequenceGroupOutput
|
||||
|
||||
|
||||
def create_output_by_sequence_group(
|
||||
outputs: GenericSequence[Union[SamplerOutput, PoolerOutput]],
|
||||
outputs: GenericSequence[SamplerOutput],
|
||||
num_seq_groups: int) -> List[List[SequenceGroupOutput]]:
|
||||
"""Helper method which transforms a 2d list organized by
|
||||
[step][sequence group] into [sequence group][step].
|
||||
"""
|
||||
output_by_sequence_group: List[List[SequenceGroupOutput]] = [
|
||||
output_by_sequence_group: List[List[CompletionSequenceGroupOutput]] = [
|
||||
[] for _ in range(num_seq_groups)
|
||||
]
|
||||
for step in outputs:
|
||||
sequence_group_output: CompletionSequenceGroupOutput
|
||||
for i, sequence_group_output in enumerate(step):
|
||||
output_by_sequence_group[i].append(sequence_group_output)
|
||||
|
||||
return output_by_sequence_group
|
||||
# Cast to the more generic type that CompletionSequenceGroupOutput
|
||||
# inherits from.
|
||||
return cast(List[List[SequenceGroupOutput]], output_by_sequence_group)
|
||||
|
||||
Reference in New Issue
Block a user