[V1][Metrics] Initial speculative decoding metrics (#15151)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
@@ -23,6 +23,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.metrics.stats import SchedulerStats
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm.v1.spec_decode.metrics import SpecDecodingStats
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -552,6 +553,7 @@ class Scheduler(SchedulerInterface):
|
||||
spec_token_ids = model_runner_output.spec_token_ids
|
||||
logprobs = model_runner_output.logprobs
|
||||
prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict
|
||||
spec_decoding_stats = SpecDecodingStats() if self.log_stats else None
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens
|
||||
|
||||
new_running: list[Request] = []
|
||||
@@ -584,6 +586,11 @@ class Scheduler(SchedulerInterface):
|
||||
len(generated_token_ids))
|
||||
request.num_computed_tokens -= num_tokens_rejected
|
||||
|
||||
if spec_decoding_stats is not None:
|
||||
spec_decoding_stats.observe(
|
||||
num_draft_tokens=len(scheduled_spec_token_ids),
|
||||
num_accepted_tokens=len(generated_token_ids) - 1)
|
||||
|
||||
cached_encoder_input_ids = (
|
||||
self.encoder_cache_manager.get_cached_input_ids(request))
|
||||
# OPTIMIZATION: Avoid list(set) if the set is empty.
|
||||
@@ -657,7 +664,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.running = new_running
|
||||
engine_core_outputs = EngineCoreOutputs(
|
||||
outputs=outputs,
|
||||
scheduler_stats=self.make_stats(),
|
||||
scheduler_stats=self.make_stats(spec_decoding_stats),
|
||||
)
|
||||
if self.include_finished_set:
|
||||
#TODO currently sending duplicates here, improve this
|
||||
@@ -724,7 +731,10 @@ class Scheduler(SchedulerInterface):
|
||||
def reset_prefix_cache(self) -> bool:
|
||||
return self.kv_cache_manager.reset_prefix_cache()
|
||||
|
||||
def make_stats(self) -> Optional[SchedulerStats]:
|
||||
def make_stats(
|
||||
self,
|
||||
spec_decoding_stats: Optional[SpecDecodingStats] = None,
|
||||
) -> Optional[SchedulerStats]:
|
||||
if not self.log_stats:
|
||||
return None
|
||||
return SchedulerStats(
|
||||
@@ -732,4 +742,5 @@ class Scheduler(SchedulerInterface):
|
||||
num_waiting_reqs=len(self.waiting),
|
||||
gpu_cache_usage=self.kv_cache_manager.usage,
|
||||
prefix_cache_stats=self.kv_cache_manager.make_prefix_cache_stats(),
|
||||
spec_decoding_stats=spec_decoding_stats,
|
||||
)
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_utils import PrefixCachingMetrics
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
|
||||
from vllm.v1.spec_decode.metrics import SpecDecodingMetrics
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -38,6 +39,7 @@ class LoggingStatLogger(StatLoggerBase):
|
||||
# Prefix cache metrics. This cannot be reset.
|
||||
# TODO: Make the interval configurable.
|
||||
self.prefix_caching_metrics = PrefixCachingMetrics()
|
||||
self.spec_decoding_metrics = SpecDecodingMetrics()
|
||||
|
||||
def _reset(self, now):
|
||||
self.last_log_time = now
|
||||
@@ -65,6 +67,10 @@ class LoggingStatLogger(StatLoggerBase):
|
||||
|
||||
self.prefix_caching_metrics.observe(scheduler_stats.prefix_cache_stats)
|
||||
|
||||
if scheduler_stats.spec_decoding_stats is not None:
|
||||
self.spec_decoding_metrics.observe(
|
||||
scheduler_stats.spec_decoding_stats)
|
||||
|
||||
self.last_scheduler_stats = scheduler_stats
|
||||
|
||||
def log(self):
|
||||
@@ -94,6 +100,9 @@ class LoggingStatLogger(StatLoggerBase):
|
||||
self.prefix_caching_metrics.hit_rate * 100,
|
||||
)
|
||||
|
||||
if scheduler_stats.spec_decoding_stats is not None:
|
||||
self.spec_decoding_metrics.log()
|
||||
|
||||
|
||||
class PrometheusStatLogger(StatLoggerBase):
|
||||
|
||||
@@ -302,6 +311,24 @@ class PrometheusStatLogger(StatLoggerBase):
|
||||
self.labelname_running_lora_adapters,
|
||||
])
|
||||
|
||||
#
|
||||
# Speculative Decoding metrics
|
||||
# The acceptance rate can be calculated using a PromQL query:
|
||||
#
|
||||
# rate(vllm:spec_decode_num_accepted_tokens_total[$interval]) /
|
||||
# rate(vllm:spec_decode_num_draft_tokens_total[$interval])
|
||||
#
|
||||
self.counter_spec_decode_num_draft_tokens = \
|
||||
prometheus_client.Counter(
|
||||
name="vllm:spec_decode_num_draft_tokens_total",
|
||||
documentation="Number of draft tokens.",
|
||||
labelnames=labelnames).labels(*labelvalues)
|
||||
self.counter_spec_decode_num_accepted_tokens = \
|
||||
prometheus_client.Counter(
|
||||
name="vllm:spec_decode_num_accepted_tokens_total",
|
||||
documentation="Number of accepted tokens.",
|
||||
labelnames=labelnames).labels(*labelvalues)
|
||||
|
||||
#
|
||||
# Cache config info metric
|
||||
#
|
||||
@@ -338,6 +365,12 @@ class PrometheusStatLogger(StatLoggerBase):
|
||||
self.counter_gpu_prefix_cache_hits.inc(
|
||||
scheduler_stats.prefix_cache_stats.hits)
|
||||
|
||||
if scheduler_stats.spec_decoding_stats is not None:
|
||||
self.counter_spec_decode_num_draft_tokens.inc(
|
||||
scheduler_stats.spec_decoding_stats.num_draft_tokens)
|
||||
self.counter_spec_decode_num_accepted_tokens.inc(
|
||||
scheduler_stats.spec_decoding_stats.num_accepted_tokens)
|
||||
|
||||
if iteration_stats is None:
|
||||
return
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from vllm.v1.spec_decode.metrics import SpecDecodingStats
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.engine import EngineCoreEvent, EngineCoreOutput, FinishReason
|
||||
from vllm.v1.engine.output_processor import RequestState
|
||||
@@ -35,6 +37,8 @@ class SchedulerStats:
|
||||
prefix_cache_stats: PrefixCacheStats = field(
|
||||
default_factory=PrefixCacheStats)
|
||||
|
||||
spec_decoding_stats: Optional[SpecDecodingStats] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoRAStats:
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SpecDecodingStats:
|
||||
num_draft_tokens: int = 0
|
||||
num_accepted_tokens: int = 0
|
||||
|
||||
def take(self):
|
||||
copied = SpecDecodingStats(self.num_draft_tokens,
|
||||
self.num_accepted_tokens)
|
||||
self.reset()
|
||||
return copied
|
||||
|
||||
def reset(self):
|
||||
self.num_draft_tokens = 0
|
||||
self.num_accepted_tokens = 0
|
||||
|
||||
def observe(self, num_draft_tokens: int, num_accepted_tokens: int):
|
||||
self.num_draft_tokens += num_draft_tokens
|
||||
self.num_accepted_tokens += num_accepted_tokens
|
||||
|
||||
|
||||
class SpecDecodingMetrics:
|
||||
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.num_draft_tokens: list[int] = []
|
||||
self.num_accepted_tokens: list[int] = []
|
||||
|
||||
def observe(self, spec_decoding_stats: SpecDecodingStats):
|
||||
self.num_draft_tokens.append(spec_decoding_stats.num_draft_tokens)
|
||||
self.num_accepted_tokens.append(
|
||||
spec_decoding_stats.num_accepted_tokens)
|
||||
|
||||
def log(self):
|
||||
num_draft_tokens = np.sum(self.num_draft_tokens)
|
||||
num_accepted_tokens = np.sum(self.num_accepted_tokens)
|
||||
|
||||
draft_acceptance_rate = (num_accepted_tokens / num_draft_tokens
|
||||
if num_draft_tokens > 0 else float("nan"))
|
||||
|
||||
logger.info(
|
||||
"Speculative metrics: "
|
||||
"Draft acceptance rate: %.3f, "
|
||||
"Number of accepted tokens: %d, "
|
||||
"Number of draft tokens: %d, ", draft_acceptance_rate,
|
||||
num_accepted_tokens, num_draft_tokens)
|
||||
self.reset()
|
||||
Reference in New Issue
Block a user