Usage Stats Collection (#2852)
This commit is contained in:
@@ -13,6 +13,7 @@ from vllm.engine.ray_utils import initialize_ray_cluster
|
||||
from vllm.executor.executor_base import ExecutorBase
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.model_loader import get_architecture_class_name
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
|
||||
@@ -21,6 +22,8 @@ from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
|
||||
from vllm.transformers_utils.detokenizer import Detokenizer
|
||||
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
|
||||
get_tokenizer_group)
|
||||
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
|
||||
usage_message)
|
||||
from vllm.utils import Counter
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -53,6 +56,7 @@ class LLMEngine:
|
||||
executor_class: The model executor class for managing distributed
|
||||
execution.
|
||||
log_stats: Whether to log statistics.
|
||||
usage_context: Specified entry point, used for usage info collection
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -66,6 +70,7 @@ class LLMEngine:
|
||||
vision_language_config: Optional["VisionLanguageConfig"],
|
||||
executor_class: Type[ExecutorBase],
|
||||
log_stats: bool,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
) -> None:
|
||||
logger.info(
|
||||
f"Initializing an LLM engine (v{vllm.__version__}) with config: "
|
||||
@@ -108,6 +113,39 @@ class LLMEngine:
|
||||
device_config, lora_config,
|
||||
vision_language_config)
|
||||
|
||||
# If usage stat is enabled, collect relevant info.
|
||||
if is_usage_stats_enabled():
|
||||
usage_message.report_usage(
|
||||
get_architecture_class_name(model_config),
|
||||
usage_context,
|
||||
extra_kvs={
|
||||
# Common configuration
|
||||
"dtype":
|
||||
str(model_config.dtype),
|
||||
"tensor_parallel_size":
|
||||
parallel_config.tensor_parallel_size,
|
||||
"block_size":
|
||||
cache_config.block_size,
|
||||
"gpu_memory_utilization":
|
||||
cache_config.gpu_memory_utilization,
|
||||
|
||||
# Quantization
|
||||
"quantization":
|
||||
model_config.quantization,
|
||||
"kv_cache_dtype":
|
||||
cache_config.cache_dtype,
|
||||
|
||||
# Feature flags
|
||||
"enable_lora":
|
||||
bool(lora_config),
|
||||
"enable_prefix_caching":
|
||||
cache_config.enable_prefix_caching,
|
||||
"enforce_eager":
|
||||
model_config.enforce_eager,
|
||||
"disable_custom_all_reduce":
|
||||
parallel_config.disable_custom_all_reduce,
|
||||
})
|
||||
|
||||
# Ping the tokenizer to ensure liveness if it runs in a
|
||||
# different process.
|
||||
self.tokenizer.ping()
|
||||
@@ -125,7 +163,11 @@ class LLMEngine:
|
||||
self.stat_logger.info("cache_config", self.cache_config)
|
||||
|
||||
@classmethod
|
||||
def from_engine_args(cls, engine_args: EngineArgs) -> "LLMEngine":
|
||||
def from_engine_args(
|
||||
cls,
|
||||
engine_args: EngineArgs,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
) -> "LLMEngine":
|
||||
"""Creates an LLM engine from the engine arguments."""
|
||||
# Create the engine configs.
|
||||
engine_configs = engine_args.create_engine_configs()
|
||||
@@ -147,9 +189,12 @@ class LLMEngine:
|
||||
executor_class = GPUExecutor
|
||||
|
||||
# Create the LLM engine.
|
||||
engine = cls(*engine_configs,
|
||||
executor_class=executor_class,
|
||||
log_stats=not engine_args.disable_log_stats)
|
||||
engine = cls(
|
||||
*engine_configs,
|
||||
executor_class=executor_class,
|
||||
log_stats=not engine_args.disable_log_stats,
|
||||
usage_context=usage_context,
|
||||
)
|
||||
return engine
|
||||
|
||||
def __reduce__(self):
|
||||
|
||||
Reference in New Issue
Block a user