[Bugfix] Enable V1 usage stats (#16986)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
co-authored by
Nick Hill
parent
46e678bcff
commit
ed50f46641
@@ -12,6 +12,8 @@ import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
|
||||
usage_message)
|
||||
from vllm.utils import get_mp_context, kill_process_tree
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -201,3 +203,45 @@ def copy_slice(from_tensor: torch.Tensor, to_tensor: torch.Tensor,
|
||||
Returns the sliced target tensor.
|
||||
"""
|
||||
return to_tensor[:length].copy_(from_tensor[:length], non_blocking=True)
|
||||
|
||||
|
||||
def report_usage_stats(vllm_config, usage_context: UsageContext) -> None:
|
||||
"""Report usage statistics if enabled."""
|
||||
|
||||
if not is_usage_stats_enabled():
|
||||
return
|
||||
|
||||
from vllm.model_executor.model_loader import get_architecture_class_name
|
||||
|
||||
usage_message.report_usage(
|
||||
get_architecture_class_name(vllm_config.model_config),
|
||||
usage_context,
|
||||
extra_kvs={
|
||||
# Common configuration
|
||||
"dtype":
|
||||
str(vllm_config.model_config.dtype),
|
||||
"tensor_parallel_size":
|
||||
vllm_config.parallel_config.tensor_parallel_size,
|
||||
"block_size":
|
||||
vllm_config.cache_config.block_size,
|
||||
"gpu_memory_utilization":
|
||||
vllm_config.cache_config.gpu_memory_utilization,
|
||||
|
||||
# Quantization
|
||||
"quantization":
|
||||
vllm_config.model_config.quantization,
|
||||
"kv_cache_dtype":
|
||||
str(vllm_config.cache_config.cache_dtype),
|
||||
|
||||
# Feature flags
|
||||
"enable_lora":
|
||||
bool(vllm_config.lora_config),
|
||||
"enable_prompt_adapter":
|
||||
bool(vllm_config.prompt_adapter_config),
|
||||
"enable_prefix_caching":
|
||||
vllm_config.cache_config.enable_prefix_caching,
|
||||
"enforce_eager":
|
||||
vllm_config.model_config.enforce_eager,
|
||||
"disable_custom_all_reduce":
|
||||
vllm_config.parallel_config.disable_custom_all_reduce,
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user