[Doc] Clarify ITL vs TPOT Prometheus metrics (#55283)

Signed-off-by: Jyotirmoy Roy <jyotirmoyroy649@gmail.com>
Signed-off-by: Cenab Batu Bora <40250311+cenab@users.noreply.github.com>
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com>
Signed-off-by: Josiah Davis <Josiah.Davis@arm.com>
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com>
Signed-off-by: Wei Zhao <weizha@oci-aga-slurm-1-vscode-02.cm.cluster>
Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Signed-off-by: JiataiWang <wangjiatai@proton.me>
Signed-off-by: 子华 <huaxi.shx@alibaba-inc.com>
Signed-off-by: Colin McNamara <colin@2cups.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Marceli Fylcek <marceli.fylcek@intel.com>
Signed-off-by: LearningMachine621 <147347705+LearningMachine621@users.noreply.github.com>
Signed-off-by: jamesETsmith <james.smith9113@gmail.com>
Signed-off-by: Cursor <cursoragent@cursor.com>
Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
Signed-off-by: wangyicong <wangyicong@bytedance.com>
Signed-off-by: Kushal <72650064+kushaldabbe@users.noreply.github.com>
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: hungh <hungh@nvidia.com>
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: zhicheng Liu <lzc021026@gmail.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Alexander Salikov <salikov57@gmail.com>
Signed-off-by: Luca Motz <luca.motz@icloud.com>
Signed-off-by: pmanczak <pawel.manczak@intel.com>
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
Signed-off-by: Julien Darve <jdarve@NVIDIA.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: khluu <khluu000@gmail.com>
Signed-off-by: Kevin H. Luu <khluu000@gmail.com>
Signed-off-by: Tony Lin <tony.lin@intel.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: JinYan Su <751080330@qq.com>
Signed-off-by: nicole-lihui <nicole.li@daocloud.io>
Signed-off-by: waizuichougou <2082431897@qq.com>
Signed-off-by: waizuichougou <105572299+waizuichougou@users.noreply.github.com>
Signed-off-by: zRzRzRzRzRzRzR <Yuxuan.Zhang2@liverpool.ac.uk>
Signed-off-by: 김재억 <gadian88@gmail.com>
Signed-off-by: xiaoming <1259730330@qq.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Guy Stone <guys@spotify.com>
Signed-off-by: Djordje Ramic <djoramic@amd.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Lucain Pouget <lucain@huggingface.co>
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
Signed-off-by: Sören Dréano <gailenstorm@gmail.com>
Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev>
Signed-off-by: Andy Lo <andy@mistral.ai>
Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Chengcheng Pei <5881383+chengchengpei@users.noreply.github.com>
Signed-off-by: Lyle Lai <lyle.lai@cienet.com>
Signed-off-by: Vincent <vincexxchan@gmail.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: xawang <xawang@amd.com>
Signed-off-by: Alec Flowers <aflowers@nvidia.com>
Signed-off-by: Fangzhou Ai <fangzhou@semianalysis.ai>
Signed-off-by: Yizheng Jiao <jyizheng@gmail.com>
Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com>
Signed-off-by: Divakar Verma <divakar.verma@amd.com>
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
Signed-off-by: Biswa Panda <biswa.panda@gmail.com>
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Signed-off-by: Canlin <canlinguosdu@gmail.com>
Signed-off-by: junuxyz <216036880+junuxyz@users.noreply.github.com>
Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Signed-off-by: arcusbuilds <srijankeshri007@gmail.com>
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Signed-off-by: Kunshang Ji <jikunshang95@gmail.com>
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Signed-off-by: JartX <sagformas@epdcenter.es>
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com>
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
Signed-off-by: Joe Cotant <joe@inferact.ai>
Signed-off-by: yangzeyu <532183776@qq.com>
Signed-off-by: Ayushman Singh <40520701+ayush1399@users.noreply.github.com>
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Simon Veitner <sveitner@redhat.com>
Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
Signed-off-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai>
Signed-off-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai>
Signed-off-by: PatrykSaffer <patryk.saffer@mistral.ai>
Signed-off-by: Chao Lei <leichao139636@163.com>
Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
Signed-off-by: Ayushman Singh <ayush1399@users.noreply.github.com>
Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Signed-off-by: surajm20061998 <surajm20061998@gmail.com>
Signed-off-by: Roger Wang <hey@rogerw.io>
Signed-off-by: Ylean <61880605+Woolgathererer@users.noreply.github.com>
Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Signed-off-by: Kebe <mail@kebe7jun.com>
Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
Signed-off-by: Shenglei Fu <sfu@confluent.io>
Signed-off-by: pavelzak <pavel.zakharov@gmail.com>
Signed-off-by: fai <fangzhouai@gmail.com>
Signed-off-by: Stu Cao <stucao@Stus-MacBook-Pro.local>
Signed-off-by: <>
Signed-off-by: Zhewen Li <zhewenli@inferact.ai>
Signed-off-by: CedricHuang <38417461+CedricHwong@users.noreply.github.com>
Signed-off-by: siyu <mg21330037@smail.nju.edu.cn>
Signed-off-by: Wang Zupeng <zupenwang@gmail.com>
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in>
Signed-off-by: Isotr0py <2037008807@qq.com>
Signed-off-by: specture724 <specture724@gmail.com>
Signed-off-by: lxy-alexander <lxy_alexander@outlook.com>
Signed-off-by: BWAAEEEK <jooho414@gmail.com>
Signed-off-by: Jared Wen <jaredwen@inferact.ai>
Signed-off-by: Chinmay Kulkarni <Chinmay.Kulkarni@amd.com>
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Signed-off-by: Jared Wen <w13431838023@gmail.com>
Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
Signed-off-by: liuqihao <liuqihao@myhexin.com>
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com>
Signed-off-by: reidliu41 <reid201711@gmail.com>
Signed-off-by: Xiaozhen Bi <xiaozhen.bi@gmail.com>
Signed-off-by: shen-shanshan <467638484@qq.com>
Signed-off-by: R <Ganesh.R@amd.com>
Signed-off-by: Alex <jihui.huang@daocloud.io>
Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: arkohut <39525455+arkohut@users.noreply.github.com>
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Signed-off-by: Orestis Zambounis <23146389+orestis-z@users.noreply.github.com>
Signed-off-by: Orestis Zambounis <orestis.zambounis@gmail.com>
Signed-off-by: Claude <noreply@anthropic.com>
Signed-off-by: Orestis Zambounis <orestis-z@users.noreply.github.com>
Signed-off-by: Speculators Bot <bot@speculators.ai>
Signed-off-by: Speculators Bot <speculatorsbot@gmail.com>
Signed-off-by: zqlcode <lizongqiang@kanzhun.com>
Signed-off-by: zql <37731799+zqlcode@users.noreply.github.com>
Signed-off-by: Thillai Chithambaram <thillaichithambaram.a@gmail.com>
Signed-off-by: ElizaWszola <ewszola@redhat.com>
Signed-off-by: snordmann <snordmann@nvidia.com>
Signed-off-by: Ianniu <985683179yzy@gmail.com>
Signed-off-by: Luossu <zhluosuu@outlook.com>
Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Signed-off-by: Michele Campi <michele.campi@outlook.com>
Signed-off-by: xiaguan <751080330@qq.com>
Signed-off-by: zhaoguochun1995 <zhaoguochun1995@163.com>
Signed-off-by: JohnQinAMD <yanyuan.qin@amd.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Liran Schour <lirans@il.ibm.com>
Signed-off-by: Liqian Lin <jacklin78911@gmail.com>
Signed-off-by: wuhangxian <1391938827@qq.com>
Signed-off-by: Yongye Zhu <yongye@inferact.ai>
Signed-off-by: Miguel Garcia <miguelgarciaroman8@gmail.com>
Signed-off-by: Mario <404mario@users.noreply.github.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
Co-authored-by: Cenab Batu Bora <40250311+cenab@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Rita Brugarolas <Rita.BrugarolasBrufau@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Josiah Davis <josiah.j.davis@gmail.com>
Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
Co-authored-by: Julien Denize <40604584+juliendenize@users.noreply.github.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Co-authored-by: Wei Zhao <weizha@oci-aga-slurm-1-vscode-02.cm.cluster>
Co-authored-by: Matt <156021403+mawong-amd@users.noreply.github.com>
Co-authored-by: Micah Williamson <micah.williamson@amd.com>
Co-authored-by: Jiatai Wang <wangjiatai@proton.me>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: shaohuaxi <huaxi.shx@alibaba-inc.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Colin McNamara <colin@2cups.com>
Co-authored-by: Marceli Fylcek <marceli.fylcek@intel.com>
Co-authored-by: 焦恺文 <15529091557@163.com>
Co-authored-by: James E T Smith <jamesETsmith@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: YukioZzz <Yichao.Zhu@amd.com>
Co-authored-by: Yicong Wang <wangyicong@bytedance.com>
Co-authored-by: Kushal <72650064+kushaldabbe@users.noreply.github.com>
Co-authored-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Hung Hoang (黄兴） <hungh@nvidia.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
Co-authored-by: Zhicheng Liu <2084597944@qq.com>
Co-authored-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Alexander <Salikov57@gmail.com>
Co-authored-by: lucamotz <luca.motz@icloud.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: pmanczak <pawel.manczak@intel.com>
Co-authored-by: Fadi Arafeh <115173828+fadara01@users.noreply.github.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
Co-authored-by: JulienDarve <86800349+JulienDarve@users.noreply.github.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Tony Lin <tony.lin@intel.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: JinYan Su <751080330@qq.com>
Co-authored-by: Nicole LiHui 🥜 <nicole.li@daocloud.io>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Co-authored-by: waizuichougou <105572299+waizuichougou@users.noreply.github.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Yuxuan Zhang <Yuxuan.Zhang2@liverpool.ac.uk>
Co-authored-by: 김재억 <gadian88@gmail.com>
Co-authored-by: labAxiaoming <34019940+labAxiaoming@users.noreply.github.com>
Co-authored-by: Guy Stone <guys@spotify.com>
Co-authored-by: djramic <djoramic@amd.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Lucain <lucain@huggingface.co>
Co-authored-by: Sören Dréano <71752785+SorenDreano@users.noreply.github.com>
Co-authored-by: Chris Eastwood <106503529+eastwood-c@users.noreply.github.com>
Co-authored-by: Andy Lo <andy@mistral.ai>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: chengchengpei <5881383+chengchengpei@users.noreply.github.com>
Co-authored-by: Lyle Lai <lyle.lai@cienet.com>
Co-authored-by: mevince <26759093+mevince@users.noreply.github.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: amd-xavierwang <xavier.wang@amd.com>
Co-authored-by: Alec <35311602+alec-flowers@users.noreply.github.com>
Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
Co-authored-by: Fangzhou Ai <fangzhou@semianalysis.ai>
Co-authored-by: Yizheng Jiao <jyizheng@gmail.com>
Co-authored-by: Tahsin Tunan <tahsintunan@gmail.com>
Co-authored-by: Divakar Verma <137818590+divakar-amd@users.noreply.github.com>
Co-authored-by: liuzhenwei <zhenweiliu@habana.ai>
Co-authored-by: Biswa Panda <biswa.panda@gmail.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
Co-authored-by: junuxyz <216036880+junuxyz@users.noreply.github.com>
Co-authored-by: Walter Beller-Morales <walterbm@users.noreply.github.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
Co-authored-by: Srijan Keshri <srijan.keshri@mindbodyonline.com>
Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: JartX <sagformas@epdcenter.es>
Co-authored-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com>
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
Co-authored-by: jcotant-inferact <joe@inferact.ai>
Co-authored-by: Zeyu Yang <532183776@qq.com>
Co-authored-by: Ayushman Singh <40520701+ayush1399@users.noreply.github.com>
Co-authored-by: Raphaël Rialland <36076211+TQCB@users.noreply.github.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
Co-authored-by: PatrykSaffer <patryk.saffer@mistral.ai>
Co-authored-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai>
Co-authored-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai>
Co-authored-by: Patryk Saffer <PatrykSaffer@users.noreply.github.com>
Co-authored-by: Chao Lei <leichao139636@163.com>
Co-authored-by: Ayushman Singh <ayush1399@users.noreply.github.com>
Co-authored-by: Markus Hartikainen <markus.hartikainen@amd.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Suraj <surajm20061998@gmail.com>
Co-authored-by: Kimi Code <noreply@moonshot.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Ylean <61880605+Woolgathererer@users.noreply.github.com>
Co-authored-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Kebe <mail@kebe7jun.com>
Co-authored-by: linitra24 <renshuang.zhou@daocloud.io>
Co-authored-by: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com>
Co-authored-by: Pavel Zakharov <pavel.zakharov@gmail.com>
Co-authored-by: stu-cao <stu@primeintellect.ai>
Co-authored-by: Stu Cao <stucao@Stus-MacBook-Pro.local>
Co-authored-by: Zhewen Li <zhewenli@meta.com>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
Co-authored-by: CedricHuang <38417461+CedricHwong@users.noreply.github.com>
Co-authored-by: siyu <mg21330037@smail.nju.edu.cn>
Co-authored-by: Zupeng Wang <zupenwang@gmail.com>
Co-authored-by: Chaojun Zhang <chaojun.zhang@intel.com>
Co-authored-by: Hotragn Pettugani <103170876+Hotragn@users.noreply.github.com>
Co-authored-by: Anjie Hou <149605198+specture724@users.noreply.github.com>
Co-authored-by: Alexander Lee <lxy_alexander@outlook.com>
Co-authored-by: JooHo Lee <96564470+BWAAEEEK@users.noreply.github.com>
Co-authored-by: Jared Wen <w13431838023@gmail.com>
Co-authored-by: Chinmay-Kulkarni-AMD <Chinmay.Kulkarni@amd.com>
Co-authored-by: Matthias Gehre <matthias.gehre@amd.com>
Co-authored-by: BruceLoveDecimal <39156883+BruceLoveDecimal@users.noreply.github.com>
Co-authored-by: liuqihao <liuqihao@myhexin.com>
Co-authored-by: Martin Hickey <martin.hickey@ie.ibm.com>
Co-authored-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Co-authored-by: Juan Pérez de Algaba <124347725+jperezdealgaba@users.noreply.github.com>
Co-authored-by: Reid <61492567+reidliu41@users.noreply.github.com>
Co-authored-by: Xiaozhen Bi <xiaozhen.bi@gmail.com>
Co-authored-by: Ganesh R <ganesh.r@amd.com>
Co-authored-by: AlexHuang <jihui.huang@daocloud.io>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: Kimi Code <noreply@moonshot.cn>
Co-authored-by: Ronald <ronaldautomobile@163.com>
Co-authored-by: arkohut <39525455+arkohut@users.noreply.github.com>
Co-authored-by: Russell Bryant <rbryant@redhat.com>
Co-authored-by: Orestis Zambounis <23146389+orestis-z@users.noreply.github.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: Orestis Zambounis <orestis-z@users.noreply.github.com>
Co-authored-by: Speculators Bot <speculatorsbot@gmail.com>
Co-authored-by: zql <37731799+zqlcode@users.noreply.github.com>
Co-authored-by: zqlcode <lizongqiang@kanzhun.com>
Co-authored-by: Thillai Chithambaram <79466435+thillai-c@users.noreply.github.com>
Co-authored-by: Mark McLoughlin <markmc@redhat.com>
Co-authored-by: Samuel Nordmann <snordmann@nvidia.com>
Co-authored-by: ElizaWszola <ewszola@redhat.com>
Co-authored-by: Zhiyao(Ian) Yu <105995086+Ianniu123@users.noreply.github.com>
Co-authored-by: 鐘天楽 <zhluosuu@outlook.com>
Co-authored-by: Jorge Gallego Feliciano <jorge.gallego@mistral.ai>
Co-authored-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Co-authored-by: sychen52 <41452870+sychen52@users.noreply.github.com>
Co-authored-by: Michele Campi <215741962+MicheleCampi@users.noreply.github.com>
Co-authored-by: Sherlock <sherlock@raft.ai>
Co-authored-by: zhaoguochun1995 <109069909+zhaoguochun1995@users.noreply.github.com>
Co-authored-by: JohnQinAMD <yanyuan.qin@amd.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Bhushan Asati <bhushanasati25@gmail.com>
Co-authored-by: Megha Agarwal <19240983+meghaagr13@users.noreply.github.com>
Co-authored-by: liranschour <liranschour@users.noreply.github.com>
Co-authored-by: Liqian Lin <jacklin78911@gmail.com>
Co-authored-by: wuhangxian <1391938827@qq.com>
Co-authored-by: Yongye Zhu <yongye@inferact.ai>
Co-authored-by: Miguel <60073809+migarci2@users.noreply.github.com>
Co-authored-by: 404mario <18063257856@163.com>
Co-authored-by: Mario <404mario@users.noreply.github.com>
Co-authored-by: Sherlock <sherlock@openai.com>
Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>

docs/design/metrics.md

@@ -33,7 +33,8 @@ In v1, an extensive set of metrics are exposed via a Prometheus-compatible `/met
 - `vllm:request_prompt_tokens` (Histogram) - Histogram of input prompt token counts.
 - `vllm:request_generation_tokens` (Histogram) - Histogram of generation token counts.
 - `vllm:time_to_first_token_seconds` (Histogram) - Time to first token (TTFT).
-- `vllm:inter_token_latency_seconds` (Histogram) - Inter-token latency.
+- `vllm:inter_token_latency_seconds` (Histogram) - Inter-token latency (time between consecutive streamed outputs).
+- `vllm:request_time_per_output_token_seconds` (Histogram) - Per-request Time Per Output Token (TPOT).
 - `vllm:e2e_request_latency_seconds` (Histogram) - End-to-end request latency.
 - `vllm:request_prefill_time_seconds` (Histogram) - Request prefill time.
 - `vllm:request_decode_time_seconds` (Histogram) - Request decode time.
@@ -49,7 +50,8 @@ The subset of metrics exposed in the Grafana dashboard gives us an indication of
 - `vllm:e2e_request_latency_seconds_bucket` - End to end request latency measured in seconds.
 - `vllm:prompt_tokens` - Prompt tokens.
 - `vllm:generation_tokens` - Generation tokens.
-- `vllm:inter_token_latency_seconds` - Inter-token latency (Time Per Output Token, TPOT) in seconds.
+- `vllm:inter_token_latency_seconds` - Inter-token latency in seconds, measured between consecutive streamed outputs. It approximates TPOT when each output contains exactly one token, but differs from request-level TPOT when an output contains multiple tokens or when aggregation weights differ.
+- `vllm:request_time_per_output_token_seconds` - Per-request Time Per Output Token (TPOT) in seconds, computed as `(end-to-end latency - TTFT) / (number of output tokens - 1)`. It is recorded as zero for requests that generate no more than one token.
 - `vllm:time_to_first_token_seconds` - Time to First Token (TTFT) latency in seconds.
 - `vllm:num_requests_running` (also, `_swapped` and `_waiting`) - Number of requests in the RUNNING, WAITING, and SWAPPED states.
 - `vllm:kv_cache_usage_perc` - Percentage of used cache blocks by vLLM.
@@ -63,6 +65,20 @@ The subset of metrics exposed in the Grafana dashboard gives us an indication of
 
 See [the PR which added this Dashboard](https://github.com/vllm-project/vllm/pull/2316) for interesting and useful background on the choices made here.
 
+`vllm:inter_token_latency_seconds` records one sample per streamed output event
+(the wall-clock gap between successive outputs), whereas
+`vllm:request_time_per_output_token_seconds` is recorded once per finished request
+as `(end-to-end latency - TTFT) / (number of output tokens - 1)`. These two
+metrics differ when an output bundles multiple tokens (e.g. speculative
+decoding) or when mean ITL and mean per-request TPOT are aggregated over
+differing weights. See [Understanding the latency metrics](../benchmarking/cli.md#understanding-the-latency-metrics)
+for the benchmark definitions. Requests that generate no more than one token
+are recorded with a TPOT of zero, while `vllm bench serve` excludes them from
+its TPOT statistics. Therefore, the Prometheus histogram and benchmark TPOT
+statistics can differ. Use `vllm:request_time_per_output_token_seconds` for
+request-level TPOT and `vllm:inter_token_latency_seconds` when you specifically
+want the inter-output latency distribution.
+
 ### Prometheus Client Library
 
 Prometheus support was initially added [using the aioprometheus library](https://github.com/vllm-project/vllm/pull/1890), but a switch was made quickly to [prometheus_client](https://github.com/vllm-project/vllm/pull/2730). The rationale is discussed in both linked PRs.
@@ -238,8 +254,7 @@ statistics relating to that iteration:
   iteration.
 - The prefill intervals for any requests that completed prefill in
   this iteration.
-- The inter-token intervals (Time Per Output Token, TPOT), for all
+- The inter-token intervals, for all requests included in this iteration.
-  requests included in this iteration.
 - The Time-To-First-Token (TTFT) for any requests that completed
   prefill in this iteration. However, we calculate this interval
   relative to when the request was first received by the frontend

examples/observability/prometheus_grafana/grafana.json

@@ -318,7 +318,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
-      "description": "Inter token latency in seconds.",
+      "description": "Inter-token latency in seconds, measured between consecutive streamed outputs. It approximates TPOT when each output contains one token, but differs from request-level TPOT under speculative decoding or varying aggregation weights.",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -479,6 +479,172 @@
       "title": "Inter Token Latency",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "Per-request Time Per Output Token (TPOT) in seconds, computed as (end-to-end latency - TTFT) divided by the number of output tokens minus one. Requests with no more than one output token are recorded as zero; vllm bench serve excludes these requests from its TPOT statistics.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 48
+      },
+      "id": 17,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:request_time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "P99",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:request_time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "P95",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:request_time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "P90",
+          "range": true,
+          "refId": "C",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:request_time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "P50",
+          "range": true,
+          "refId": "D",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "rate(vllm:request_time_per_output_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:request_time_per_output_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Mean",
+          "range": true,
+          "refId": "E"
+        }
+      ],
+      "title": "Time Per Output Token (TPOT)",
+      "type": "timeseries"
+    },
     {
       "datasource": {
         "type": "prometheus",