Usage Stats Collection (#2852)

This commit is contained in:
yhu422
2024-03-28 22:16:12 -07:00
committed by GitHub
parent 7bc94a0fdd
commit d8658c8cc1
15 changed files with 362 additions and 24 deletions
+3 -1
View File
@@ -10,6 +10,7 @@ from vllm.lora.request import LoRARequest
from vllm.outputs import RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.sequence import MultiModalData
from vllm.usage.usage_lib import UsageContext
from vllm.utils import Counter
@@ -108,7 +109,8 @@ class LLM:
disable_custom_all_reduce=disable_custom_all_reduce,
**kwargs,
)
self.llm_engine = LLMEngine.from_engine_args(engine_args)
self.llm_engine = LLMEngine.from_engine_args(
engine_args, usage_context=UsageContext.LLM_CLASS)
self.request_counter = Counter()
def get_tokenizer(