[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)
Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Reed Meyerson
Claude Opus 4.8
parent
2f41c894e3
commit
7cfb97e337
@@ -10,6 +10,7 @@ from vllm.config import (
|
||||
ECTransferConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
ObservabilityConfig,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
@@ -71,6 +72,7 @@ def create_scheduler(
|
||||
ec_role: str | None = None,
|
||||
use_v2_model_runner: bool | None = None,
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
per_request_spec_decode_metrics: str = "none",
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Create scheduler under test.
|
||||
|
||||
@@ -175,6 +177,9 @@ def create_scheduler(
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
speculative_config=speculative_config,
|
||||
ec_transfer_config=ec_transfer_config,
|
||||
observability_config=ObservabilityConfig(
|
||||
per_request_spec_decode_metrics=per_request_spec_decode_metrics,
|
||||
),
|
||||
)
|
||||
if kv_cache_spec is None:
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
|
||||
Reference in New Issue
Block a user