diff --git a/benchmarks/sonnet.txt b/benchmarks/sonnet.txt deleted file mode 100644 index 34c444e8ce..0000000000 --- a/benchmarks/sonnet.txt +++ /dev/null @@ -1,518 +0,0 @@ -FROM fairest creatures we desire increase, -That thereby beauty's rose might never die, -But as the riper should by time decease, -His tender heir might bear his memory: -But thou, contracted to thine own bright eyes, -Feed'st thy light'st flame with self-substantial fuel, -Making a famine where abundance lies, -Thyself thy foe, to thy sweet self too cruel. -Thou that art now the world's fresh ornament -And only herald to the gaudy spring, -Within thine own bud buriest thy content -And, tender churl, makest waste in niggarding. -Pity the world, or else this glutton be, -To eat the world's due, by the grave and thee. -When forty winters shall beseige thy brow, -And dig deep trenches in thy beauty's field, -Thy youth's proud livery, so gazed on now, -Will be a tatter'd weed, of small worth held: -Then being ask'd where all thy beauty lies, -Where all the treasure of thy lusty days, -To say, within thine own deep-sunken eyes, -Were an all-eating shame and thriftless praise. -How much more praise deserved thy beauty's use, -If thou couldst answer 'This fair child of mine -Shall sum my count and make my old excuse,' -Proving his beauty by succession thine! -This were to be new made when thou art old, -And see thy blood warm when thou feel'st it cold. -Look in thy glass, and tell the face thou viewest -Now is the time that face should form another; -Whose fresh repair if now thou not renewest, -Thou dost beguile the world, unbless some mother. -For where is she so fair whose unear'd womb -Disdains the tillage of thy husbandry? -Or who is he so fond will be the tomb -Of his self-love, to stop posterity? -Thou art thy mother's glass, and she in thee -Calls back the lovely April of her prime: -So thou through windows of thine age shall see -Despite of wrinkles this thy golden time. -But if thou live, remember'd not to be, -Die single, and thine image dies with thee. -Unthrifty loveliness, why dost thou spend -Upon thyself thy beauty's legacy? -Nature's bequest gives nothing but doth lend, -And being frank she lends to those are free. -Then, beauteous niggard, why dost thou abuse -The bounteous largess given thee to give? -Profitless usurer, why dost thou use -So great a sum of sums, yet canst not live? -For having traffic with thyself alone, -Thou of thyself thy sweet self dost deceive. -Then how, when nature calls thee to be gone, -What acceptable audit canst thou leave? -Thy unused beauty must be tomb'd with thee, -Which, used, lives th' executor to be. -Those hours, that with gentle work did frame -The lovely gaze where every eye doth dwell, -Will play the tyrants to the very same -And that unfair which fairly doth excel: -For never-resting time leads summer on -To hideous winter and confounds him there; -Sap cheque'd with frost and lusty leaves quite gone, -Beauty o'ersnow'd and bareness every where: -Then, were not summer's distillation left, -A liquid prisoner pent in walls of glass, -Beauty's effect with beauty were bereft, -Nor it nor no remembrance what it was: -But flowers distill'd though they with winter meet, -Leese but their show; their substance still lives sweet. -Then let not winter's ragged hand deface -In thee thy summer, ere thou be distill'd: -Make sweet some vial; treasure thou some place -With beauty's treasure, ere it be self-kill'd. -That use is not forbidden usury, -Which happies those that pay the willing loan; -That's for thyself to breed another thee, -Or ten times happier, be it ten for one; -Ten times thyself were happier than thou art, -If ten of thine ten times refigured thee: -Then what could death do, if thou shouldst depart, -Leaving thee living in posterity? -Be not self-will'd, for thou art much too fair -To be death's conquest and make worms thine heir. -Lo! in the orient when the gracious light -Lifts up his burning head, each under eye -Doth homage to his new-appearing sight, -Serving with looks his sacred majesty; -And having climb'd the steep-up heavenly hill, -Resembling strong youth in his middle age, -yet mortal looks adore his beauty still, -Attending on his golden pilgrimage; -But when from highmost pitch, with weary car, -Like feeble age, he reeleth from the day, -The eyes, 'fore duteous, now converted are -From his low tract and look another way: -So thou, thyself out-going in thy noon, -Unlook'd on diest, unless thou get a son. -Music to hear, why hear'st thou music sadly? -Sweets with sweets war not, joy delights in joy. -Why lovest thou that which thou receivest not gladly, -Or else receivest with pleasure thine annoy? -If the true concord of well-tuned sounds, -By unions married, do offend thine ear, -They do but sweetly chide thee, who confounds -In singleness the parts that thou shouldst bear. -Mark how one string, sweet husband to another, -Strikes each in each by mutual ordering, -Resembling sire and child and happy mother -Who all in one, one pleasing note do sing: -Whose speechless song, being many, seeming one, -Sings this to thee: 'thou single wilt prove none.' -Is it for fear to wet a widow's eye -That thou consumest thyself in single life? -Ah! if thou issueless shalt hap to die. -The world will wail thee, like a makeless wife; -The world will be thy widow and still weep -That thou no form of thee hast left behind, -When every private widow well may keep -By children's eyes her husband's shape in mind. -Look, what an unthrift in the world doth spend -Shifts but his place, for still the world enjoys it; -But beauty's waste hath in the world an end, -And kept unused, the user so destroys it. -No love toward others in that bosom sits -That on himself such murderous shame commits. -For shame! deny that thou bear'st love to any, -Who for thyself art so unprovident. -Grant, if thou wilt, thou art beloved of many, -But that thou none lovest is most evident; -For thou art so possess'd with murderous hate -That 'gainst thyself thou stick'st not to conspire. -Seeking that beauteous roof to ruinate -Which to repair should be thy chief desire. -O, change thy thought, that I may change my mind! -Shall hate be fairer lodged than gentle love? -Be, as thy presence is, gracious and kind, -Or to thyself at least kind-hearted prove: -Make thee another self, for love of me, -That beauty still may live in thine or thee. -As fast as thou shalt wane, so fast thou growest -In one of thine, from that which thou departest; -And that fresh blood which youngly thou bestowest -Thou mayst call thine when thou from youth convertest. -Herein lives wisdom, beauty and increase: -Without this, folly, age and cold decay: -If all were minded so, the times should cease -And threescore year would make the world away. -Let those whom Nature hath not made for store, -Harsh featureless and rude, barrenly perish: -Look, whom she best endow'd she gave the more; -Which bounteous gift thou shouldst in bounty cherish: -She carved thee for her seal, and meant thereby -Thou shouldst print more, not let that copy die. -When I do count the clock that tells the time, -And see the brave day sunk in hideous night; -When I behold the violet past prime, -And sable curls all silver'd o'er with white; -When lofty trees I see barren of leaves -Which erst from heat did canopy the herd, -And summer's green all girded up in sheaves -Borne on the bier with white and bristly beard, -Then of thy beauty do I question make, -That thou among the wastes of time must go, -Since sweets and beauties do themselves forsake -And die as fast as they see others grow; -And nothing 'gainst Time's scythe can make defence -Save breed, to brave him when he takes thee hence. -O, that you were yourself! but, love, you are -No longer yours than you yourself here live: -Against this coming end you should prepare, -And your sweet semblance to some other give. -So should that beauty which you hold in lease -Find no determination: then you were -Yourself again after yourself's decease, -When your sweet issue your sweet form should bear. -Who lets so fair a house fall to decay, -Which husbandry in honour might uphold -Against the stormy gusts of winter's day -And barren rage of death's eternal cold? -O, none but unthrifts! Dear my love, you know -You had a father: let your son say so. -Not from the stars do I my judgment pluck; -And yet methinks I have astronomy, -But not to tell of good or evil luck, -Of plagues, of dearths, or seasons' quality; -Nor can I fortune to brief minutes tell, -Pointing to each his thunder, rain and wind, -Or say with princes if it shall go well, -By oft predict that I in heaven find: -But from thine eyes my knowledge I derive, -And, constant stars, in them I read such art -As truth and beauty shall together thrive, -If from thyself to store thou wouldst convert; -Or else of thee this I prognosticate: -Thy end is truth's and beauty's doom and date. -When I consider every thing that grows -Holds in perfection but a little moment, -That this huge stage presenteth nought but shows -Whereon the stars in secret influence comment; -When I perceive that men as plants increase, -Cheered and cheque'd even by the self-same sky, -Vaunt in their youthful sap, at height decrease, -And wear their brave state out of memory; -Then the conceit of this inconstant stay -Sets you most rich in youth before my sight, -Where wasteful Time debateth with Decay, -To change your day of youth to sullied night; -And all in war with Time for love of you, -As he takes from you, I engraft you new. -But wherefore do not you a mightier way -Make war upon this bloody tyrant, Time? -And fortify yourself in your decay -With means more blessed than my barren rhyme? -Now stand you on the top of happy hours, -And many maiden gardens yet unset -With virtuous wish would bear your living flowers, -Much liker than your painted counterfeit: -So should the lines of life that life repair, -Which this, Time's pencil, or my pupil pen, -Neither in inward worth nor outward fair, -Can make you live yourself in eyes of men. -To give away yourself keeps yourself still, -And you must live, drawn by your own sweet skill. -Who will believe my verse in time to come, -If it were fill'd with your most high deserts? -Though yet, heaven knows, it is but as a tomb -Which hides your life and shows not half your parts. -If I could write the beauty of your eyes -And in fresh numbers number all your graces, -The age to come would say 'This poet lies: -Such heavenly touches ne'er touch'd earthly faces.' -So should my papers yellow'd with their age -Be scorn'd like old men of less truth than tongue, -And your true rights be term'd a poet's rage -And stretched metre of an antique song: -But were some child of yours alive that time, -You should live twice; in it and in my rhyme. -Shall I compare thee to a summer's day? -Thou art more lovely and more temperate: -Rough winds do shake the darling buds of May, -And summer's lease hath all too short a date: -Sometime too hot the eye of heaven shines, -And often is his gold complexion dimm'd; -And every fair from fair sometime declines, -By chance or nature's changing course untrimm'd; -But thy eternal summer shall not fade -Nor lose possession of that fair thou owest; -Nor shall Death brag thou wander'st in his shade, -When in eternal lines to time thou growest: -So long as men can breathe or eyes can see, -So long lives this and this gives life to thee. -Devouring Time, blunt thou the lion's paws, -And make the earth devour her own sweet brood; -Pluck the keen teeth from the fierce tiger's jaws, -And burn the long-lived phoenix in her blood; -Make glad and sorry seasons as thou fleets, -And do whate'er thou wilt, swift-footed Time, -To the wide world and all her fading sweets; -But I forbid thee one most heinous crime: -O, carve not with thy hours my love's fair brow, -Nor draw no lines there with thine antique pen; -Him in thy course untainted do allow -For beauty's pattern to succeeding men. -Yet, do thy worst, old Time: despite thy wrong, -My love shall in my verse ever live young. -A woman's face with Nature's own hand painted -Hast thou, the master-mistress of my passion; -A woman's gentle heart, but not acquainted -With shifting change, as is false women's fashion; -An eye more bright than theirs, less false in rolling, -Gilding the object whereupon it gazeth; -A man in hue, all 'hues' in his controlling, -Much steals men's eyes and women's souls amazeth. -And for a woman wert thou first created; -Till Nature, as she wrought thee, fell a-doting, -And by addition me of thee defeated, -By adding one thing to my purpose nothing. -But since she prick'd thee out for women's pleasure, -Mine be thy love and thy love's use their treasure. -So is it not with me as with that Muse -Stirr'd by a painted beauty to his verse, -Who heaven itself for ornament doth use -And every fair with his fair doth rehearse -Making a couplement of proud compare, -With sun and moon, with earth and sea's rich gems, -With April's first-born flowers, and all things rare -That heaven's air in this huge rondure hems. -O' let me, true in love, but truly write, -And then believe me, my love is as fair -As any mother's child, though not so bright -As those gold candles fix'd in heaven's air: -Let them say more than like of hearsay well; -I will not praise that purpose not to sell. -My glass shall not persuade me I am old, -So long as youth and thou are of one date; -But when in thee time's furrows I behold, -Then look I death my days should expiate. -For all that beauty that doth cover thee -Is but the seemly raiment of my heart, -Which in thy breast doth live, as thine in me: -How can I then be elder than thou art? -O, therefore, love, be of thyself so wary -As I, not for myself, but for thee will; -Bearing thy heart, which I will keep so chary -As tender nurse her babe from faring ill. -Presume not on thy heart when mine is slain; -Thou gavest me thine, not to give back again. -As an unperfect actor on the stage -Who with his fear is put besides his part, -Or some fierce thing replete with too much rage, -Whose strength's abundance weakens his own heart. -So I, for fear of trust, forget to say -The perfect ceremony of love's rite, -And in mine own love's strength seem to decay, -O'ercharged with burden of mine own love's might. -O, let my books be then the eloquence -And dumb presagers of my speaking breast, -Who plead for love and look for recompense -More than that tongue that more hath more express'd. -O, learn to read what silent love hath writ: -To hear with eyes belongs to love's fine wit. -Mine eye hath play'd the painter and hath stell'd -Thy beauty's form in table of my heart; -My body is the frame wherein 'tis held, -And perspective it is the painter's art. -For through the painter must you see his skill, -To find where your true image pictured lies; -Which in my bosom's shop is hanging still, -That hath his windows glazed with thine eyes. -Now see what good turns eyes for eyes have done: -Mine eyes have drawn thy shape, and thine for me -Are windows to my breast, where-through the sun -Delights to peep, to gaze therein on thee; -Yet eyes this cunning want to grace their art; -They draw but what they see, know not the heart. -Let those who are in favour with their stars -Of public honour and proud titles boast, -Whilst I, whom fortune of such triumph bars, -Unlook'd for joy in that I honour most. -Great princes' favourites their fair leaves spread -But as the marigold at the sun's eye, -And in themselves their pride lies buried, -For at a frown they in their glory die. -The painful warrior famoused for fight, -After a thousand victories once foil'd, -Is from the book of honour razed quite, -And all the rest forgot for which he toil'd: -Then happy I, that love and am beloved -Where I may not remove nor be removed. -Lord of my love, to whom in vassalage -Thy merit hath my duty strongly knit, -To thee I send this written embassage, -To witness duty, not to show my wit: -Duty so great, which wit so poor as mine -May make seem bare, in wanting words to show it, -But that I hope some good conceit of thine -In thy soul's thought, all naked, will bestow it; -Till whatsoever star that guides my moving -Points on me graciously with fair aspect -And puts apparel on my tatter'd loving, -To show me worthy of thy sweet respect: -Then may I dare to boast how I do love thee; -Till then not show my head where thou mayst prove me. -Weary with toil, I haste me to my bed, -The dear repose for limbs with travel tired; -But then begins a journey in my head, -To work my mind, when body's work's expired: -For then my thoughts, from far where I abide, -Intend a zealous pilgrimage to thee, -And keep my drooping eyelids open wide, -Looking on darkness which the blind do see -Save that my soul's imaginary sight -Presents thy shadow to my sightless view, -Which, like a jewel hung in ghastly night, -Makes black night beauteous and her old face new. -Lo! thus, by day my limbs, by night my mind, -For thee and for myself no quiet find. -How can I then return in happy plight, -That am debarr'd the benefit of rest? -When day's oppression is not eased by night, -But day by night, and night by day, oppress'd? -And each, though enemies to either's reign, -Do in consent shake hands to torture me; -The one by toil, the other to complain -How far I toil, still farther off from thee. -I tell the day, to please them thou art bright -And dost him grace when clouds do blot the heaven: -So flatter I the swart-complexion'd night, -When sparkling stars twire not thou gild'st the even. -But day doth daily draw my sorrows longer -And night doth nightly make grief's strength seem stronger. -When, in disgrace with fortune and men's eyes, -I all alone beweep my outcast state -And trouble deal heaven with my bootless cries -And look upon myself and curse my fate, -Wishing me like to one more rich in hope, -Featured like him, like him with friends possess'd, -Desiring this man's art and that man's scope, -With what I most enjoy contented least; -Yet in these thoughts myself almost despising, -Haply I think on thee, and then my state, -Like to the lark at break of day arising -From sullen earth, sings hymns at heaven's gate; -For thy sweet love remember'd such wealth brings -That then I scorn to change my state with kings. -When to the sessions of sweet silent thought -I summon up remembrance of things past, -I sigh the lack of many a thing I sought, -And with old woes new wail my dear time's waste: -Then can I drown an eye, unused to flow, -For precious friends hid in death's dateless night, -And weep afresh love's long since cancell'd woe, -And moan the expense of many a vanish'd sight: -Then can I grieve at grievances foregone, -And heavily from woe to woe tell o'er -The sad account of fore-bemoaned moan, -Which I new pay as if not paid before. -But if the while I think on thee, dear friend, -All losses are restored and sorrows end. -Thy bosom is endeared with all hearts, -Which I by lacking have supposed dead, -And there reigns love and all love's loving parts, -And all those friends which I thought buried. -How many a holy and obsequious tear -Hath dear religious love stol'n from mine eye -As interest of the dead, which now appear -But things removed that hidden in thee lie! -Thou art the grave where buried love doth live, -Hung with the trophies of my lovers gone, -Who all their parts of me to thee did give; -That due of many now is thine alone: -Their images I loved I view in thee, -And thou, all they, hast all the all of me. -If thou survive my well-contented day, -When that churl Death my bones with dust shall cover, -And shalt by fortune once more re-survey -These poor rude lines of thy deceased lover, -Compare them with the bettering of the time, -And though they be outstripp'd by every pen, -Reserve them for my love, not for their rhyme, -Exceeded by the height of happier men. -O, then vouchsafe me but this loving thought: -'Had my friend's Muse grown with this growing age, -A dearer birth than this his love had brought, -To march in ranks of better equipage: -But since he died and poets better prove, -Theirs for their style I'll read, his for his love.' -Full many a glorious morning have I seen -Flatter the mountain-tops with sovereign eye, -Kissing with golden face the meadows green, -Gilding pale streams with heavenly alchemy; -Anon permit the basest clouds to ride -With ugly rack on his celestial face, -And from the forlorn world his visage hide, -Stealing unseen to west with this disgrace: -Even so my sun one early morn did shine -With all triumphant splendor on my brow; -But out, alack! he was but one hour mine; -The region cloud hath mask'd him from me now. -Yet him for this my love no whit disdaineth; -Suns of the world may stain when heaven's sun staineth. -Why didst thou promise such a beauteous day, -And make me travel forth without my cloak, -To let base clouds o'ertake me in my way, -Hiding thy bravery in their rotten smoke? -'Tis not enough that through the cloud thou break, -To dry the rain on my storm-beaten face, -For no man well of such a salve can speak -That heals the wound and cures not the disgrace: -Nor can thy shame give physic to my grief; -Though thou repent, yet I have still the loss: -The offender's sorrow lends but weak relief -To him that bears the strong offence's cross. -Ah! but those tears are pearl which thy love sheds, -And they are rich and ransom all ill deeds. -No more be grieved at that which thou hast done: -Roses have thorns, and silver fountains mud; -Clouds and eclipses stain both moon and sun, -And loathsome canker lives in sweetest bud. -All men make faults, and even I in this, -Authorizing thy trespass with compare, -Myself corrupting, salving thy amiss, -Excusing thy sins more than thy sins are; -For to thy sensual fault I bring in sense-- -Thy adverse party is thy advocate-- -And 'gainst myself a lawful plea commence: -Such civil war is in my love and hate -That I an accessary needs must be -To that sweet thief which sourly robs from me. -Let me confess that we two must be twain, -Although our undivided loves are one: -So shall those blots that do with me remain -Without thy help by me be borne alone. -In our two loves there is but one respect, -Though in our lives a separable spite, -Which though it alter not love's sole effect, -Yet doth it steal sweet hours from love's delight. -I may not evermore acknowledge thee, -Lest my bewailed guilt should do thee shame, -Nor thou with public kindness honour me, -Unless thou take that honour from thy name: -But do not so; I love thee in such sort -As, thou being mine, mine is thy good report. -As a decrepit father takes delight -To see his active child do deeds of youth, -So I, made lame by fortune's dearest spite, -Take all my comfort of thy worth and truth. -For whether beauty, birth, or wealth, or wit, -Or any of these all, or all, or more, -Entitled in thy parts do crowned sit, -I make my love engrafted to this store: -So then I am not lame, poor, nor despised, -Whilst that this shadow doth such substance give -That I in thy abundance am sufficed -And by a part of all thy glory live. -Look, what is best, that best I wish in thee: -This wish I have; then ten times happy me! \ No newline at end of file diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index d05f0c233e..a1001149aa 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -23,7 +23,6 @@ th { | ShareGPT4V (Image) | ✅ | ✅ | `wget https://huggingface.co/datasets/Lin-Chen/ShareGPT4V/resolve/main/sharegpt4v_instruct_gpt4-vision_cap100k.json`
Note that the images need to be downloaded separately. For example, to download COCO's 2017 Train images:
`wget http://images.cocodataset.org/zips/train2017.zip` | | ShareGPT4Video (Video) | ✅ | ✅ | `git clone https://huggingface.co/datasets/ShareGPT4Video/ShareGPT4Video` | | BurstGPT | ✅ | ✅ | `wget https://github.com/HPMLL/BurstGPT/releases/download/v1.1/BurstGPT_without_fails_2.csv` | -| Sonnet (deprecated) | ✅ | ✅ | Local file: `benchmarks/sonnet.txt` | | Random | ✅ | ✅ | `synthetic` | | RandomMultiModal (Image/Video) | ✅ | ✅ | `synthetic` | | RandomForReranking | ✅ | ✅ | `synthetic` | @@ -773,16 +772,17 @@ Using KV cache metrics for load pattern configuration: ```bash vllm bench throughput \ --model NousResearch/Hermes-3-Llama-3.1-8B \ - --dataset-name sonnet \ - --dataset-path vllm/benchmarks/sonnet.txt \ + --dataset-name random \ + --random-input-len 500 \ + --random-output-len 150 \ --num-prompts 10 ``` If successful, you will see the following output ```text -Throughput: 7.15 requests/s, 4656.00 total tokens/s, 1072.15 output tokens/s -Total num prompt tokens: 5014 +Throughput: 7.15 requests/s, 4647.50 total tokens/s, 1072.50 output tokens/s +Total num prompt tokens: 5000 Total num output tokens: 1500 ``` diff --git a/pyproject.toml b/pyproject.toml index 890dd6bdb7..53ce6dfbe0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -192,7 +192,6 @@ python = "./.venv" [tool.typos.files] # these files may be written in non english words extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizers_/*", - "benchmarks/sonnet.txt", "rust/src/bench/src/datasets/sonnet.txt", "tests/lora/data/*", "build/*", "examples/pooling/token_embed/*", "tests/models/language/pooling/*", "vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", diff --git a/rust/src/bench/AGENTS.md b/rust/src/bench/AGENTS.md index d9a17f63f2..5cf33e0858 100644 --- a/rust/src/bench/AGENTS.md +++ b/rust/src/bench/AGENTS.md @@ -47,7 +47,6 @@ cargo test -p vllm-bench -- --ignored - `src/datasets/random.rs` — Random dataset generation with rayon parallelism - `src/datasets/random_mm.rs` — Random multimodal dataset (synthetic JPEG images, bucket config sampling, pre-serialized JSON fragments); `--enable-multimodal-chat` pre-builds the chat `messages` array at dataset time (mirrors Python's `apply_multimodal_chat_transformation`) - `src/datasets/sharegpt.rs` — ShareGPT JSON loader + HuggingFace Hub auto-download with caching -- `src/datasets/sonnet.rs` and `src/datasets/sonnet.txt` — Sonnet dataset (built-in Shakespeare sonnets via `include_str!("sonnet.txt")`; controllable token length + shared prefix; mirrors Python `SonnetDataset`) - `src/datasets/speed_bench.rs` — NVIDIA SPEED-Bench loader (HF datasets-server API, 6 configs, 11 categories, local cache) - `src/datasets/hf_dataset.rs` — Generic HuggingFace dataset loader (parquet shards via hf-hub into the standard HF hub cache; datasets-server for config/split discovery; native main-branch parquet/JSON fallback for private datasets; column auto-detection) - `src/datasets/custom.rs` — Custom JSONL dataset (`{"prompt": ..., "output_tokens": ...}` per line; `--custom-output-len -1` uses per-line output_tokens; prompts always sent raw — no client-side chat template) diff --git a/rust/src/bench/README.md b/rust/src/bench/README.md index fb1e5b529d..043605d9c8 100644 --- a/rust/src/bench/README.md +++ b/rust/src/bench/README.md @@ -13,7 +13,7 @@ vllm-bench --backend vllm --base-url http://127.0.0.1:8000 \ - **Fast** — ~7 ms startup, single ~7 MB static binary, no Python imports. - **Scales** — `Arc` prompt sharing + mimalloc keep memory <100 MB at 1400+ concurrency. -- **Many datasets** — `random`, `random-mm` (VLM), `sharegpt`, `sonnet`, `speed-bench`, and any HuggingFace dataset. +- **Many datasets** — `random`, `random-mm` (VLM), `sharegpt`, `speed-bench`, and any HuggingFace dataset. - **Many backends** — completions, chat, embeddings, pooling, and rerank. - **Beyond a single run** — concurrency/rate **sweeps**, **multi-run** stats, **multi-turn** conversations, **LoRA** multi-adapter, and result **comparison**. - **Steady-state metrics** — throughput/latency measured over the saturated plateau, excluding ramp-up and drain. @@ -127,7 +127,7 @@ vllm-bench \
-Datasets (ShareGPT / Sonnet / HuggingFace / SPEED-Bench) +Datasets (ShareGPT / HuggingFace / SPEED-Bench) ```bash # ShareGPT (auto-downloads from HuggingFace on first run, cached afterwards) @@ -141,14 +141,6 @@ vllm-bench \ --dataset-name sharegpt --dataset-path /path/to/ShareGPT_V3.json \ --num-prompts 500 --save-result -# Sonnet — built-in Shakespeare sonnets, no dataset file needed. -# Generates prompts of a controllable token length with a shared prefix. -vllm-bench \ - --backend openai-chat --base-url http://127.0.0.1:8000 --model \ - --dataset-name sonnet \ - --sonnet-input-len 550 --sonnet-output-len 150 --sonnet-prefix-len 200 \ - --num-prompts 500 - # Any HuggingFace dataset (downloads parquet shards into the standard HF hub # cache — shared with `hf download` and Python tooling — auto-detects columns) vllm-bench \ @@ -449,7 +441,7 @@ to measure a concurrent serving-style workload. | `vllm-pooling` | `/pooling` | vLLM native pooling endpoint | | `vllm-rerank` | `/v1/rerank` | vLLM reranking (query from prompt, documents via `--extra-body`) | -Pooling backends are non-streaming and report E2EL (end-to-end latency) only. Use `--dataset-name sharegpt`, `sonnet`, or `hf` for text-based embedding/rerank benchmarks, or `random` for token-ID-based embedding benchmarks. +Pooling backends are non-streaming and report E2EL (end-to-end latency) only. Use `--dataset-name sharegpt` or `hf` for text-based embedding/rerank benchmarks, or `random` for token-ID-based embedding benchmarks. ## Supported Datasets @@ -458,7 +450,6 @@ Pooling backends are non-streaming and report E2EL (end-to-end latency) only. Us | `random` | Synthetic prompts with exact token-length matching (default) | | `random-mm` | Synthetic multimodal prompts with random JPEG images for VLM benchmarking (requires `openai-chat`) | | `sharegpt` | Real conversations from ShareGPT (auto-downloads from HuggingFace, or use `--dataset-path`) | -| `sonnet` | Built-in Shakespeare sonnets; controllable token length + shared prefix, no dataset file needed | | `speed-bench` | NVIDIA SPEED-Bench for speculative decoding evaluation (auto-downloads, 11 categories) | | `hf` | Any HuggingFace dataset with parquet data (downloads parquet shards via hf-hub into the standard HF hub cache, auto-detects chat/text columns) | @@ -523,8 +514,8 @@ Run `vllm-bench --help` for the authoritative list. Grouped reference below. | Flag | Default | Description | | ------ | --------- | ------------- | -| `--dataset-name` | `random` | Dataset type (`random`, `random-mm`, `sharegpt`, `sonnet`, `speed-bench`, `hf`) | -| `--dataset-path` | — | Path to dataset file (optional for `sharegpt`/`sonnet`, which auto-source) | +| `--dataset-name` | `random` | Dataset type (`random`, `random-mm`, `sharegpt`, `speed-bench`, `hf`) | +| `--dataset-path` | — | Path to dataset file (`sharegpt` auto-downloads when omitted) | | `--num-prompts` | `1000` | Number of prompts to generate (conversations in multi-turn mode) | | `--max-model-len` | — | Filter out requests where `prompt_len + output_len` exceeds this context length | | `--input-len` | — | Override input length (general) | @@ -545,10 +536,6 @@ Run `vllm-bench --help` for the authoritative list. Grouped reference below. | `--random-mm-bucket-config` | `{(256,256,1): 0.5, (720,1280,1): 0.5}` | `(height,width,frames)` → probability (Python tuple syntax; frames=1 = image) | | **ShareGPT** | | | | `--sharegpt-output-len` | — | Override output length | -| **Sonnet** | | | -| `--sonnet-input-len` | `550` | Input tokens per request | -| `--sonnet-output-len` | `150` | Output tokens per request | -| `--sonnet-prefix-len` | `200` | Prefix tokens shared across requests | | **SPEED-Bench** | | | | `--speed-bench-config` | `qualitative` | Split (`qualitative`, `throughput_1k`/`2k`/`8k`/`16k`/`32k`); alias: `--speed-bench-dataset-subset` (Python name) | | `--speed-bench-output-len` | `4096` | Output tokens per request (matches Python default) | @@ -817,7 +804,6 @@ src/ │ ├── random_mm.rs # Random multimodal dataset (JPEG generation, bucket sampling) │ ├── multi_turn.rs # Multi-turn synthetic + ShareGPT conversation generators │ ├── sharegpt.rs # ShareGPT JSON dataset loader -│ ├── sonnet.rs # Sonnet dataset (built-in Shakespeare sonnets) │ ├── speed_bench.rs # NVIDIA SPEED-Bench loader (auto-download + cache) │ └── hf_dataset.rs # Generic HuggingFace dataset (auto-download, column detection) ├── metrics/ diff --git a/rust/src/bench/src/benchmark.rs b/rust/src/bench/src/benchmark.rs index 54b6e8701f..161646fe65 100644 --- a/rust/src/bench/src/benchmark.rs +++ b/rust/src/bench/src/benchmark.rs @@ -370,14 +370,6 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result { config.num_prompts, config.dataset_path.as_deref().unwrap_or("auto-download") ), - DatasetName::Sonnet => format!( - "{} prompts from Sonnet ({}, isl={}, osl={}, prefix={})", - config.num_prompts, - config.dataset_path.as_deref().unwrap_or("built-in"), - config.sonnet_input_len, - config.sonnet_output_len, - config.sonnet_prefix_len, - ), DatasetName::SpeedBench => { let truncate_info = config .speed_bench_max_input_len @@ -484,21 +476,6 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result { config.disable_shuffle, )? } - DatasetName::Sonnet => { - let tok = tokenizer - .as_ref() - .ok_or_else(|| BenchError::Config("Sonnet dataset requires a tokenizer".into()))?; - crate::datasets::sonnet::load_sonnet_dataset( - tok, - config.dataset_path.as_deref(), - config.num_prompts, - config.sonnet_input_len, - config.sonnet_output_len, - config.sonnet_prefix_len, - config.seed, - &config.request_id_prefix, - )? - } DatasetName::SpeedBench => { let tok = tokenizer.as_ref().ok_or_else(|| { BenchError::Config("SPEED-Bench dataset requires a tokenizer".into()) diff --git a/rust/src/bench/src/cli.rs b/rust/src/bench/src/cli.rs index 302dbb4792..4782818ccb 100644 --- a/rust/src/bench/src/cli.rs +++ b/rust/src/bench/src/cli.rs @@ -73,8 +73,6 @@ pub enum DatasetName { RandomMm, #[value(name = "sharegpt")] ShareGpt, - #[value(name = "sonnet")] - Sonnet, #[value(name = "speed-bench", alias = "speed_bench")] SpeedBench, #[value(name = "hf")] @@ -521,19 +519,6 @@ pub struct BenchServeArgs { #[arg(long, default_value_t = false)] pub disable_shuffle: bool, - // --- Sonnet dataset --- - /// Number of input tokens per request (sonnet dataset). - #[arg(long, default_value_t = crate::datasets::sonnet::DEFAULT_INPUT_LEN)] - pub sonnet_input_len: usize, - - /// Number of output tokens per request (sonnet dataset). - #[arg(long, default_value_t = crate::datasets::sonnet::DEFAULT_OUTPUT_LEN)] - pub sonnet_output_len: usize, - - /// Number of prefix tokens shared across requests (sonnet dataset). - #[arg(long, default_value_t = crate::datasets::sonnet::DEFAULT_PREFIX_LEN)] - pub sonnet_prefix_len: usize, - /// SPEED-Bench config/split (qualitative, throughput_1k, throughput_2k, throughput_8k, /// throughput_16k, throughput_32k). #[arg( diff --git a/rust/src/bench/src/config.rs b/rust/src/bench/src/config.rs index 84f55ad427..197cf08e7d 100644 --- a/rust/src/bench/src/config.rs +++ b/rust/src/bench/src/config.rs @@ -145,9 +145,6 @@ pub struct BenchConfig { pub prefix_repetition_num_prefixes: usize, pub prefix_repetition_output_len: usize, pub sharegpt_output_len: Option, - pub sonnet_input_len: usize, - pub sonnet_output_len: usize, - pub sonnet_prefix_len: usize, pub no_oversample: bool, pub disable_shuffle: bool, pub num_prompts: usize, @@ -447,8 +444,7 @@ impl BenchConfig { (MmLimitPerPrompt::default(), Vec::new()) }; - // Note: --dataset-path is optional for sharegpt (auto-downloads) and - // sonnet (uses built-in Shakespeare's sonnets). + // --dataset-path is optional for sharegpt (auto-downloads). // Range ratio (Python semantics: [len*(1-r), len*(1+r)], each r in [0,1)) let random_range_ratio = RangeRatio::parse(&args.random_range_ratio)?; @@ -689,9 +685,6 @@ impl BenchConfig { random_cache_hit_fraction: args.random_cache_hit_fraction, random_cache_ratio: args.random_cache_ratio, sharegpt_output_len: args.sharegpt_output_len, - sonnet_input_len: args.sonnet_input_len, - sonnet_output_len: args.sonnet_output_len, - sonnet_prefix_len: args.sonnet_prefix_len, no_oversample: args.no_oversample, disable_shuffle: args.disable_shuffle, num_prompts: args.num_prompts, diff --git a/rust/src/bench/src/datasets/mod.rs b/rust/src/bench/src/datasets/mod.rs index 8f81f8d3c7..4848ec5303 100644 --- a/rust/src/bench/src/datasets/mod.rs +++ b/rust/src/bench/src/datasets/mod.rs @@ -10,7 +10,6 @@ pub mod random; pub mod random_mm; pub mod random_rerank; pub mod sharegpt; -pub mod sonnet; pub mod speed_bench; use std::sync::Arc; diff --git a/rust/src/bench/src/datasets/sonnet.rs b/rust/src/bench/src/datasets/sonnet.rs deleted file mode 100644 index 6fc7a75c0b..0000000000 --- a/rust/src/bench/src/datasets/sonnet.rs +++ /dev/null @@ -1,131 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -use std::sync::Arc; - -use rand::SeedableRng; -use rand::rngs::StdRng; -use rand::seq::IndexedRandom; - -use super::SampleRequest; -use crate::error::{BenchError, Result}; -use crate::tokenizer::TokenizerKind; - -/// Default values mirror Python's SonnetDataset defaults (datasets.py). -pub const DEFAULT_PREFIX_LEN: usize = 200; -pub const DEFAULT_INPUT_LEN: usize = 550; -pub const DEFAULT_OUTPUT_LEN: usize = 150; - -const BASE_PROMPT: &str = "Pick as many lines as you can from these poem lines:\n"; - -/// Shakespeare's sonnets, public domain. Bundled so `--dataset-name sonnet` works -/// out of the box without `--dataset-path`. Source: -/// https://raw.githubusercontent.com/vllm-project/vllm/main/benchmarks/sonnet.txt -const BUILTIN_SONNET: &str = include_str!("sonnet.txt"); - -/// Load the sonnet dataset and generate `num_requests` prompts targeting `input_len` -/// total prompt tokens. -/// -/// Mirrors Python's `SonnetDataset.sample()` from `vllm/benchmarks/datasets/datasets.py`. -/// The Rust port skips `apply_chat_template` (no Jinja runtime here): `base_offset` -/// and `prompt_len` are computed from the raw text. The resulting prompts are slightly -/// shorter than Python's chat-template-formatted version (off by the chat scaffolding -/// tokens, typically <20). -pub fn load_sonnet_dataset( - tokenizer: &TokenizerKind, - dataset_path: Option<&str>, - num_requests: usize, - input_len: usize, - output_len: usize, - prefix_len: usize, - seed: u64, - request_id_prefix: &str, -) -> Result> { - let content = match dataset_path { - Some(path) => std::fs::read_to_string(path) - .map_err(|e| BenchError::Config(format!("Failed to read sonnet file '{path}': {e}")))?, - None => BUILTIN_SONNET.to_string(), - }; - - // Match Python's f.readlines(): keep trailing newlines so that joining lines - // reconstructs the original text without inserting extra separators. - let lines: Vec = content.split_inclusive('\n').map(|s| s.to_string()).collect(); - - if lines.is_empty() { - let src = dataset_path.unwrap_or(""); - return Err(BenchError::Config(format!("Sonnet file '{src}' is empty"))); - } - - // Average tokens per line (used to estimate how many lines to draw). - let mut total_tokens: usize = 0; - for line in &lines { - let ids = tokenizer.encode(line, false)?; - total_tokens += ids.len(); - } - let avg_len = total_tokens as f64 / lines.len() as f64; - if avg_len <= 0.0 { - return Err(BenchError::Config( - "Sonnet lines tokenized to zero tokens on average".into(), - )); - } - - let base_ids = tokenizer.encode(BASE_PROMPT, false)?; - let base_offset = base_ids.len(); - if input_len <= base_offset { - return Err(BenchError::Config(format!( - "--sonnet-input-len ({input_len}) must be larger than the base prompt length ({base_offset})" - ))); - } - - let num_input_lines = ((input_len - base_offset) as f64 / avg_len).round() as i64; - let num_prefix_lines = - (((prefix_len as i64 - base_offset as i64) as f64) / avg_len).round().max(0.0) as i64; - let num_input_lines = num_input_lines.max(0) as usize; - let num_prefix_lines = (num_prefix_lines as usize).min(lines.len()); - let num_input_lines = num_input_lines.max(num_prefix_lines); - - let prefix_lines: &[String] = &lines[..num_prefix_lines]; - let extras_per_request = num_input_lines - num_prefix_lines; - - let mut rng = StdRng::seed_from_u64(seed); - let mut samples = Vec::with_capacity(num_requests); - let mut ind = 0usize; - let mut attempts = 0usize; - let max_attempts = num_requests.saturating_mul(20).max(1000); - - while samples.len() < num_requests { - if attempts >= max_attempts { - return Err(BenchError::Config(format!( - "Could not assemble {num_requests} sonnet prompts under input_len={input_len} \ - after {attempts} attempts. Try increasing --sonnet-input-len." - ))); - } - attempts += 1; - - let mut prompt = String::with_capacity(BASE_PROMPT.len() + 256 * num_input_lines); - prompt.push_str(BASE_PROMPT); - for line in prefix_lines { - prompt.push_str(line); - } - for _ in 0..extras_per_request { - // random.choices with replacement — duplicates are allowed. - let line = lines.choose(&mut rng).unwrap(); - prompt.push_str(line); - } - - let prompt_ids = tokenizer.encode(&prompt, false)?; - let prompt_len = prompt_ids.len(); - if prompt_len <= input_len { - samples.push(SampleRequest { - prompt: Arc::from(prompt), - prompt_len, - expected_output_len: output_len, - request_id: Some(format!("{request_id_prefix}{ind}")), - ..Default::default() - }); - ind += 1; - } - } - - Ok(samples) -} diff --git a/rust/src/bench/src/datasets/sonnet.txt b/rust/src/bench/src/datasets/sonnet.txt deleted file mode 100644 index 34c444e8ce..0000000000 --- a/rust/src/bench/src/datasets/sonnet.txt +++ /dev/null @@ -1,518 +0,0 @@ -FROM fairest creatures we desire increase, -That thereby beauty's rose might never die, -But as the riper should by time decease, -His tender heir might bear his memory: -But thou, contracted to thine own bright eyes, -Feed'st thy light'st flame with self-substantial fuel, -Making a famine where abundance lies, -Thyself thy foe, to thy sweet self too cruel. -Thou that art now the world's fresh ornament -And only herald to the gaudy spring, -Within thine own bud buriest thy content -And, tender churl, makest waste in niggarding. -Pity the world, or else this glutton be, -To eat the world's due, by the grave and thee. -When forty winters shall beseige thy brow, -And dig deep trenches in thy beauty's field, -Thy youth's proud livery, so gazed on now, -Will be a tatter'd weed, of small worth held: -Then being ask'd where all thy beauty lies, -Where all the treasure of thy lusty days, -To say, within thine own deep-sunken eyes, -Were an all-eating shame and thriftless praise. -How much more praise deserved thy beauty's use, -If thou couldst answer 'This fair child of mine -Shall sum my count and make my old excuse,' -Proving his beauty by succession thine! -This were to be new made when thou art old, -And see thy blood warm when thou feel'st it cold. -Look in thy glass, and tell the face thou viewest -Now is the time that face should form another; -Whose fresh repair if now thou not renewest, -Thou dost beguile the world, unbless some mother. -For where is she so fair whose unear'd womb -Disdains the tillage of thy husbandry? -Or who is he so fond will be the tomb -Of his self-love, to stop posterity? -Thou art thy mother's glass, and she in thee -Calls back the lovely April of her prime: -So thou through windows of thine age shall see -Despite of wrinkles this thy golden time. -But if thou live, remember'd not to be, -Die single, and thine image dies with thee. -Unthrifty loveliness, why dost thou spend -Upon thyself thy beauty's legacy? -Nature's bequest gives nothing but doth lend, -And being frank she lends to those are free. -Then, beauteous niggard, why dost thou abuse -The bounteous largess given thee to give? -Profitless usurer, why dost thou use -So great a sum of sums, yet canst not live? -For having traffic with thyself alone, -Thou of thyself thy sweet self dost deceive. -Then how, when nature calls thee to be gone, -What acceptable audit canst thou leave? -Thy unused beauty must be tomb'd with thee, -Which, used, lives th' executor to be. -Those hours, that with gentle work did frame -The lovely gaze where every eye doth dwell, -Will play the tyrants to the very same -And that unfair which fairly doth excel: -For never-resting time leads summer on -To hideous winter and confounds him there; -Sap cheque'd with frost and lusty leaves quite gone, -Beauty o'ersnow'd and bareness every where: -Then, were not summer's distillation left, -A liquid prisoner pent in walls of glass, -Beauty's effect with beauty were bereft, -Nor it nor no remembrance what it was: -But flowers distill'd though they with winter meet, -Leese but their show; their substance still lives sweet. -Then let not winter's ragged hand deface -In thee thy summer, ere thou be distill'd: -Make sweet some vial; treasure thou some place -With beauty's treasure, ere it be self-kill'd. -That use is not forbidden usury, -Which happies those that pay the willing loan; -That's for thyself to breed another thee, -Or ten times happier, be it ten for one; -Ten times thyself were happier than thou art, -If ten of thine ten times refigured thee: -Then what could death do, if thou shouldst depart, -Leaving thee living in posterity? -Be not self-will'd, for thou art much too fair -To be death's conquest and make worms thine heir. -Lo! in the orient when the gracious light -Lifts up his burning head, each under eye -Doth homage to his new-appearing sight, -Serving with looks his sacred majesty; -And having climb'd the steep-up heavenly hill, -Resembling strong youth in his middle age, -yet mortal looks adore his beauty still, -Attending on his golden pilgrimage; -But when from highmost pitch, with weary car, -Like feeble age, he reeleth from the day, -The eyes, 'fore duteous, now converted are -From his low tract and look another way: -So thou, thyself out-going in thy noon, -Unlook'd on diest, unless thou get a son. -Music to hear, why hear'st thou music sadly? -Sweets with sweets war not, joy delights in joy. -Why lovest thou that which thou receivest not gladly, -Or else receivest with pleasure thine annoy? -If the true concord of well-tuned sounds, -By unions married, do offend thine ear, -They do but sweetly chide thee, who confounds -In singleness the parts that thou shouldst bear. -Mark how one string, sweet husband to another, -Strikes each in each by mutual ordering, -Resembling sire and child and happy mother -Who all in one, one pleasing note do sing: -Whose speechless song, being many, seeming one, -Sings this to thee: 'thou single wilt prove none.' -Is it for fear to wet a widow's eye -That thou consumest thyself in single life? -Ah! if thou issueless shalt hap to die. -The world will wail thee, like a makeless wife; -The world will be thy widow and still weep -That thou no form of thee hast left behind, -When every private widow well may keep -By children's eyes her husband's shape in mind. -Look, what an unthrift in the world doth spend -Shifts but his place, for still the world enjoys it; -But beauty's waste hath in the world an end, -And kept unused, the user so destroys it. -No love toward others in that bosom sits -That on himself such murderous shame commits. -For shame! deny that thou bear'st love to any, -Who for thyself art so unprovident. -Grant, if thou wilt, thou art beloved of many, -But that thou none lovest is most evident; -For thou art so possess'd with murderous hate -That 'gainst thyself thou stick'st not to conspire. -Seeking that beauteous roof to ruinate -Which to repair should be thy chief desire. -O, change thy thought, that I may change my mind! -Shall hate be fairer lodged than gentle love? -Be, as thy presence is, gracious and kind, -Or to thyself at least kind-hearted prove: -Make thee another self, for love of me, -That beauty still may live in thine or thee. -As fast as thou shalt wane, so fast thou growest -In one of thine, from that which thou departest; -And that fresh blood which youngly thou bestowest -Thou mayst call thine when thou from youth convertest. -Herein lives wisdom, beauty and increase: -Without this, folly, age and cold decay: -If all were minded so, the times should cease -And threescore year would make the world away. -Let those whom Nature hath not made for store, -Harsh featureless and rude, barrenly perish: -Look, whom she best endow'd she gave the more; -Which bounteous gift thou shouldst in bounty cherish: -She carved thee for her seal, and meant thereby -Thou shouldst print more, not let that copy die. -When I do count the clock that tells the time, -And see the brave day sunk in hideous night; -When I behold the violet past prime, -And sable curls all silver'd o'er with white; -When lofty trees I see barren of leaves -Which erst from heat did canopy the herd, -And summer's green all girded up in sheaves -Borne on the bier with white and bristly beard, -Then of thy beauty do I question make, -That thou among the wastes of time must go, -Since sweets and beauties do themselves forsake -And die as fast as they see others grow; -And nothing 'gainst Time's scythe can make defence -Save breed, to brave him when he takes thee hence. -O, that you were yourself! but, love, you are -No longer yours than you yourself here live: -Against this coming end you should prepare, -And your sweet semblance to some other give. -So should that beauty which you hold in lease -Find no determination: then you were -Yourself again after yourself's decease, -When your sweet issue your sweet form should bear. -Who lets so fair a house fall to decay, -Which husbandry in honour might uphold -Against the stormy gusts of winter's day -And barren rage of death's eternal cold? -O, none but unthrifts! Dear my love, you know -You had a father: let your son say so. -Not from the stars do I my judgment pluck; -And yet methinks I have astronomy, -But not to tell of good or evil luck, -Of plagues, of dearths, or seasons' quality; -Nor can I fortune to brief minutes tell, -Pointing to each his thunder, rain and wind, -Or say with princes if it shall go well, -By oft predict that I in heaven find: -But from thine eyes my knowledge I derive, -And, constant stars, in them I read such art -As truth and beauty shall together thrive, -If from thyself to store thou wouldst convert; -Or else of thee this I prognosticate: -Thy end is truth's and beauty's doom and date. -When I consider every thing that grows -Holds in perfection but a little moment, -That this huge stage presenteth nought but shows -Whereon the stars in secret influence comment; -When I perceive that men as plants increase, -Cheered and cheque'd even by the self-same sky, -Vaunt in their youthful sap, at height decrease, -And wear their brave state out of memory; -Then the conceit of this inconstant stay -Sets you most rich in youth before my sight, -Where wasteful Time debateth with Decay, -To change your day of youth to sullied night; -And all in war with Time for love of you, -As he takes from you, I engraft you new. -But wherefore do not you a mightier way -Make war upon this bloody tyrant, Time? -And fortify yourself in your decay -With means more blessed than my barren rhyme? -Now stand you on the top of happy hours, -And many maiden gardens yet unset -With virtuous wish would bear your living flowers, -Much liker than your painted counterfeit: -So should the lines of life that life repair, -Which this, Time's pencil, or my pupil pen, -Neither in inward worth nor outward fair, -Can make you live yourself in eyes of men. -To give away yourself keeps yourself still, -And you must live, drawn by your own sweet skill. -Who will believe my verse in time to come, -If it were fill'd with your most high deserts? -Though yet, heaven knows, it is but as a tomb -Which hides your life and shows not half your parts. -If I could write the beauty of your eyes -And in fresh numbers number all your graces, -The age to come would say 'This poet lies: -Such heavenly touches ne'er touch'd earthly faces.' -So should my papers yellow'd with their age -Be scorn'd like old men of less truth than tongue, -And your true rights be term'd a poet's rage -And stretched metre of an antique song: -But were some child of yours alive that time, -You should live twice; in it and in my rhyme. -Shall I compare thee to a summer's day? -Thou art more lovely and more temperate: -Rough winds do shake the darling buds of May, -And summer's lease hath all too short a date: -Sometime too hot the eye of heaven shines, -And often is his gold complexion dimm'd; -And every fair from fair sometime declines, -By chance or nature's changing course untrimm'd; -But thy eternal summer shall not fade -Nor lose possession of that fair thou owest; -Nor shall Death brag thou wander'st in his shade, -When in eternal lines to time thou growest: -So long as men can breathe or eyes can see, -So long lives this and this gives life to thee. -Devouring Time, blunt thou the lion's paws, -And make the earth devour her own sweet brood; -Pluck the keen teeth from the fierce tiger's jaws, -And burn the long-lived phoenix in her blood; -Make glad and sorry seasons as thou fleets, -And do whate'er thou wilt, swift-footed Time, -To the wide world and all her fading sweets; -But I forbid thee one most heinous crime: -O, carve not with thy hours my love's fair brow, -Nor draw no lines there with thine antique pen; -Him in thy course untainted do allow -For beauty's pattern to succeeding men. -Yet, do thy worst, old Time: despite thy wrong, -My love shall in my verse ever live young. -A woman's face with Nature's own hand painted -Hast thou, the master-mistress of my passion; -A woman's gentle heart, but not acquainted -With shifting change, as is false women's fashion; -An eye more bright than theirs, less false in rolling, -Gilding the object whereupon it gazeth; -A man in hue, all 'hues' in his controlling, -Much steals men's eyes and women's souls amazeth. -And for a woman wert thou first created; -Till Nature, as she wrought thee, fell a-doting, -And by addition me of thee defeated, -By adding one thing to my purpose nothing. -But since she prick'd thee out for women's pleasure, -Mine be thy love and thy love's use their treasure. -So is it not with me as with that Muse -Stirr'd by a painted beauty to his verse, -Who heaven itself for ornament doth use -And every fair with his fair doth rehearse -Making a couplement of proud compare, -With sun and moon, with earth and sea's rich gems, -With April's first-born flowers, and all things rare -That heaven's air in this huge rondure hems. -O' let me, true in love, but truly write, -And then believe me, my love is as fair -As any mother's child, though not so bright -As those gold candles fix'd in heaven's air: -Let them say more than like of hearsay well; -I will not praise that purpose not to sell. -My glass shall not persuade me I am old, -So long as youth and thou are of one date; -But when in thee time's furrows I behold, -Then look I death my days should expiate. -For all that beauty that doth cover thee -Is but the seemly raiment of my heart, -Which in thy breast doth live, as thine in me: -How can I then be elder than thou art? -O, therefore, love, be of thyself so wary -As I, not for myself, but for thee will; -Bearing thy heart, which I will keep so chary -As tender nurse her babe from faring ill. -Presume not on thy heart when mine is slain; -Thou gavest me thine, not to give back again. -As an unperfect actor on the stage -Who with his fear is put besides his part, -Or some fierce thing replete with too much rage, -Whose strength's abundance weakens his own heart. -So I, for fear of trust, forget to say -The perfect ceremony of love's rite, -And in mine own love's strength seem to decay, -O'ercharged with burden of mine own love's might. -O, let my books be then the eloquence -And dumb presagers of my speaking breast, -Who plead for love and look for recompense -More than that tongue that more hath more express'd. -O, learn to read what silent love hath writ: -To hear with eyes belongs to love's fine wit. -Mine eye hath play'd the painter and hath stell'd -Thy beauty's form in table of my heart; -My body is the frame wherein 'tis held, -And perspective it is the painter's art. -For through the painter must you see his skill, -To find where your true image pictured lies; -Which in my bosom's shop is hanging still, -That hath his windows glazed with thine eyes. -Now see what good turns eyes for eyes have done: -Mine eyes have drawn thy shape, and thine for me -Are windows to my breast, where-through the sun -Delights to peep, to gaze therein on thee; -Yet eyes this cunning want to grace their art; -They draw but what they see, know not the heart. -Let those who are in favour with their stars -Of public honour and proud titles boast, -Whilst I, whom fortune of such triumph bars, -Unlook'd for joy in that I honour most. -Great princes' favourites their fair leaves spread -But as the marigold at the sun's eye, -And in themselves their pride lies buried, -For at a frown they in their glory die. -The painful warrior famoused for fight, -After a thousand victories once foil'd, -Is from the book of honour razed quite, -And all the rest forgot for which he toil'd: -Then happy I, that love and am beloved -Where I may not remove nor be removed. -Lord of my love, to whom in vassalage -Thy merit hath my duty strongly knit, -To thee I send this written embassage, -To witness duty, not to show my wit: -Duty so great, which wit so poor as mine -May make seem bare, in wanting words to show it, -But that I hope some good conceit of thine -In thy soul's thought, all naked, will bestow it; -Till whatsoever star that guides my moving -Points on me graciously with fair aspect -And puts apparel on my tatter'd loving, -To show me worthy of thy sweet respect: -Then may I dare to boast how I do love thee; -Till then not show my head where thou mayst prove me. -Weary with toil, I haste me to my bed, -The dear repose for limbs with travel tired; -But then begins a journey in my head, -To work my mind, when body's work's expired: -For then my thoughts, from far where I abide, -Intend a zealous pilgrimage to thee, -And keep my drooping eyelids open wide, -Looking on darkness which the blind do see -Save that my soul's imaginary sight -Presents thy shadow to my sightless view, -Which, like a jewel hung in ghastly night, -Makes black night beauteous and her old face new. -Lo! thus, by day my limbs, by night my mind, -For thee and for myself no quiet find. -How can I then return in happy plight, -That am debarr'd the benefit of rest? -When day's oppression is not eased by night, -But day by night, and night by day, oppress'd? -And each, though enemies to either's reign, -Do in consent shake hands to torture me; -The one by toil, the other to complain -How far I toil, still farther off from thee. -I tell the day, to please them thou art bright -And dost him grace when clouds do blot the heaven: -So flatter I the swart-complexion'd night, -When sparkling stars twire not thou gild'st the even. -But day doth daily draw my sorrows longer -And night doth nightly make grief's strength seem stronger. -When, in disgrace with fortune and men's eyes, -I all alone beweep my outcast state -And trouble deal heaven with my bootless cries -And look upon myself and curse my fate, -Wishing me like to one more rich in hope, -Featured like him, like him with friends possess'd, -Desiring this man's art and that man's scope, -With what I most enjoy contented least; -Yet in these thoughts myself almost despising, -Haply I think on thee, and then my state, -Like to the lark at break of day arising -From sullen earth, sings hymns at heaven's gate; -For thy sweet love remember'd such wealth brings -That then I scorn to change my state with kings. -When to the sessions of sweet silent thought -I summon up remembrance of things past, -I sigh the lack of many a thing I sought, -And with old woes new wail my dear time's waste: -Then can I drown an eye, unused to flow, -For precious friends hid in death's dateless night, -And weep afresh love's long since cancell'd woe, -And moan the expense of many a vanish'd sight: -Then can I grieve at grievances foregone, -And heavily from woe to woe tell o'er -The sad account of fore-bemoaned moan, -Which I new pay as if not paid before. -But if the while I think on thee, dear friend, -All losses are restored and sorrows end. -Thy bosom is endeared with all hearts, -Which I by lacking have supposed dead, -And there reigns love and all love's loving parts, -And all those friends which I thought buried. -How many a holy and obsequious tear -Hath dear religious love stol'n from mine eye -As interest of the dead, which now appear -But things removed that hidden in thee lie! -Thou art the grave where buried love doth live, -Hung with the trophies of my lovers gone, -Who all their parts of me to thee did give; -That due of many now is thine alone: -Their images I loved I view in thee, -And thou, all they, hast all the all of me. -If thou survive my well-contented day, -When that churl Death my bones with dust shall cover, -And shalt by fortune once more re-survey -These poor rude lines of thy deceased lover, -Compare them with the bettering of the time, -And though they be outstripp'd by every pen, -Reserve them for my love, not for their rhyme, -Exceeded by the height of happier men. -O, then vouchsafe me but this loving thought: -'Had my friend's Muse grown with this growing age, -A dearer birth than this his love had brought, -To march in ranks of better equipage: -But since he died and poets better prove, -Theirs for their style I'll read, his for his love.' -Full many a glorious morning have I seen -Flatter the mountain-tops with sovereign eye, -Kissing with golden face the meadows green, -Gilding pale streams with heavenly alchemy; -Anon permit the basest clouds to ride -With ugly rack on his celestial face, -And from the forlorn world his visage hide, -Stealing unseen to west with this disgrace: -Even so my sun one early morn did shine -With all triumphant splendor on my brow; -But out, alack! he was but one hour mine; -The region cloud hath mask'd him from me now. -Yet him for this my love no whit disdaineth; -Suns of the world may stain when heaven's sun staineth. -Why didst thou promise such a beauteous day, -And make me travel forth without my cloak, -To let base clouds o'ertake me in my way, -Hiding thy bravery in their rotten smoke? -'Tis not enough that through the cloud thou break, -To dry the rain on my storm-beaten face, -For no man well of such a salve can speak -That heals the wound and cures not the disgrace: -Nor can thy shame give physic to my grief; -Though thou repent, yet I have still the loss: -The offender's sorrow lends but weak relief -To him that bears the strong offence's cross. -Ah! but those tears are pearl which thy love sheds, -And they are rich and ransom all ill deeds. -No more be grieved at that which thou hast done: -Roses have thorns, and silver fountains mud; -Clouds and eclipses stain both moon and sun, -And loathsome canker lives in sweetest bud. -All men make faults, and even I in this, -Authorizing thy trespass with compare, -Myself corrupting, salving thy amiss, -Excusing thy sins more than thy sins are; -For to thy sensual fault I bring in sense-- -Thy adverse party is thy advocate-- -And 'gainst myself a lawful plea commence: -Such civil war is in my love and hate -That I an accessary needs must be -To that sweet thief which sourly robs from me. -Let me confess that we two must be twain, -Although our undivided loves are one: -So shall those blots that do with me remain -Without thy help by me be borne alone. -In our two loves there is but one respect, -Though in our lives a separable spite, -Which though it alter not love's sole effect, -Yet doth it steal sweet hours from love's delight. -I may not evermore acknowledge thee, -Lest my bewailed guilt should do thee shame, -Nor thou with public kindness honour me, -Unless thou take that honour from thy name: -But do not so; I love thee in such sort -As, thou being mine, mine is thy good report. -As a decrepit father takes delight -To see his active child do deeds of youth, -So I, made lame by fortune's dearest spite, -Take all my comfort of thy worth and truth. -For whether beauty, birth, or wealth, or wit, -Or any of these all, or all, or more, -Entitled in thy parts do crowned sit, -I make my love engrafted to this store: -So then I am not lame, poor, nor despised, -Whilst that this shadow doth such substance give -That I in thy abundance am sufficed -And by a part of all thy glory live. -Look, what is best, that best I wish in thee: -This wish I have; then ten times happy me! \ No newline at end of file diff --git a/rust/src/bench/src/multi_turn.rs b/rust/src/bench/src/multi_turn.rs index 366fd4f6fb..b2b39b14cf 100644 --- a/rust/src/bench/src/multi_turn.rs +++ b/rust/src/bench/src/multi_turn.rs @@ -170,11 +170,6 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result { - return Err(BenchError::Config( - "Sonnet multi-turn is not yet supported. Use 'random' or 'sharegpt' with --multi-turn.".into(), - )); - } DatasetName::SpeedBench => { return Err(BenchError::Config( "SPEED-Bench multi-turn is not yet supported. Use 'random' or 'sharegpt' with --multi-turn.".into(), diff --git a/rust/src/bench/tests/python_serve_flags.txt b/rust/src/bench/tests/python_serve_flags.txt index d200e5bf24..3206196178 100644 --- a/rust/src/bench/tests/python_serve_flags.txt +++ b/rust/src/bench/tests/python_serve_flags.txt @@ -86,9 +86,6 @@ --sharegpt-output-len --skip-chat-template --skip-tokenizer-init ---sonnet-input-len ---sonnet-output-len ---sonnet-prefix-len --spec-bench-category --spec-bench-output-len --speed-bench-category diff --git a/vllm/benchmarks/datasets/__init__.py b/vllm/benchmarks/datasets/__init__.py index b003ee4c05..f1f58954b0 100644 --- a/vllm/benchmarks/datasets/__init__.py +++ b/vllm/benchmarks/datasets/__init__.py @@ -27,7 +27,6 @@ from vllm.benchmarks.datasets.datasets import ( RandomMultiModalDataset, SampleRequest, ShareGPTDataset, - SonnetDataset, SpecBench, VisionArenaDataset, add_dataset_parser, @@ -71,7 +70,6 @@ __all__ = [ "RandomMultiModalDataset", "SampleRequest", "ShareGPTDataset", - "SonnetDataset", "SpecBench", "VisionArenaDataset", "add_dataset_parser", diff --git a/vllm/benchmarks/datasets/create_txt_slices_dataset.py b/vllm/benchmarks/datasets/create_txt_slices_dataset.py index 1bcd93c8ae..ed3523a653 100644 --- a/vllm/benchmarks/datasets/create_txt_slices_dataset.py +++ b/vllm/benchmarks/datasets/create_txt_slices_dataset.py @@ -12,8 +12,8 @@ Usage :: python -m vllm.benchmarks.datasets.create_txt_slices_dataset \\ - --input sonnet.txt \\ - --output sonnet_dataset.jsonl \\ + --input corpus.txt \\ + --output dataset.jsonl \\ --tokenizer gpt2 \\ --num-prompts 1000 \\ --input-len 1024 \\ @@ -23,7 +23,7 @@ The resulting JSONL file can then be used with the serving benchmark:: python -m vllm.benchmarks.serve \\ --dataset-name custom \\ - --dataset-path sonnet_dataset.jsonl \\ + --dataset-path dataset.jsonl \\ ... """ diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index abaf299a19..3746ac727e 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -5,7 +5,6 @@ datasets. Each dataset subclass of BenchmarkDataset must implement sample generation. Supported dataset types include: - ShareGPT - Random (synthetic) - - Sonnet - BurstGPT - HuggingFace - VisionArena @@ -31,7 +30,6 @@ from typing import Any, cast import numpy as np import pybase64 as base64 from PIL import Image -from typing_extensions import deprecated from vllm.benchmarks.datasets.utils import ( RangeRatio, @@ -1598,7 +1596,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser): choices=[ "sharegpt", "burstgpt", - "sonnet", "random", "random-mm", "random-rerank", @@ -1622,8 +1619,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser): "--dataset-path", type=str, default=None, - help="Path to the sharegpt/sonnet dataset or the HF dataset ID if " - "using HF dataset.", + help="Path to the ShareGPT dataset or the HF dataset ID if using HF dataset.", ) parser.add_argument( "--no-oversample", @@ -1681,26 +1677,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser): help="Category for spec bench dataset. If None, use all categories.", ) - sonnet_group = parser.add_argument_group("sonnet dataset options") - sonnet_group.add_argument( - "--sonnet-input-len", - type=int, - default=550, - help="Number of input tokens per request, used only for sonnet dataset.", - ) - sonnet_group.add_argument( - "--sonnet-output-len", - type=int, - default=150, - help="Number of output tokens per request, used only for sonnet dataset.", - ) - sonnet_group.add_argument( - "--sonnet-prefix-len", - type=int, - default=200, - help="Number of prefix tokens per request, used only for sonnet dataset.", - ) - sharegpt_group = parser.add_argument_group("sharegpt dataset options") sharegpt_group.add_argument( "--sharegpt-output-len", @@ -2134,45 +2110,6 @@ def get_samples( no_oversample=args.no_oversample, ) - elif args.dataset_name == "sonnet": - assert tokenizer is not None, ( - "Tokenizer must be initialized for the 'sonnet' dataset." - ) - sonnet_dataset = SonnetDataset( - dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle - ) - # For the "sonnet" dataset, formatting depends on the backend. - # Chat-style backends leave templating to the server; completions - # backends need the prompt rendered client side. - if args.backend in ("openai-chat", "openai-responses"): - input_requests = sonnet_dataset.sample( - num_requests=args.num_prompts, - input_len=args.sonnet_input_len, - output_len=args.sonnet_output_len, - prefix_len=args.sonnet_prefix_len, - tokenizer=tokenizer, - return_prompt_formatted=False, - request_id_prefix=args.request_id_prefix, - no_oversample=args.no_oversample, - ) - else: - assert ( - hasattr(tokenizer, "chat_template") and tokenizer.chat_template - ) or ( - hasattr(tokenizer, "default_chat_template") - and tokenizer.default_chat_template - ), "Tokenizer/model must have chat template for sonnet dataset." - input_requests = sonnet_dataset.sample( - num_requests=args.num_prompts, - input_len=args.sonnet_input_len, - output_len=args.sonnet_output_len, - prefix_len=args.sonnet_prefix_len, - tokenizer=tokenizer, - return_prompt_formatted=True, - request_id_prefix=args.request_id_prefix, - no_oversample=args.no_oversample, - ) - elif args.dataset_name == "hf": # all following datasets are implemented from the # HuggingFaceDataset base class @@ -3070,106 +3007,6 @@ class SpecBench(CustomDataset): ) -# ----------------------------------------------------------------------------- -# Sonnet Dataset Implementation -# ----------------------------------------------------------------------------- - - -@deprecated( - "SonnetDataset is deprecated and will be removed in a future version.", -) -class SonnetDataset(BenchmarkDataset): - """Simplified implementation of the Sonnet dataset. Loads poem lines from a - text file and generates sample requests. Default values here copied from - `benchmark_serving.py` for the sonnet dataset. - """ - - DEFAULT_PREFIX_LEN = 200 - DEFAULT_INPUT_LEN = 550 - DEFAULT_OUTPUT_LEN = 150 - - def __init__( - self, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.load_data() - - def load_data(self) -> None: - if not self.dataset_path: - raise ValueError("dataset_path must be provided.") - with open(self.dataset_path, encoding="utf-8") as f: - self.data = f.readlines() - - def sample( - self, - tokenizer: TokenizerLike, - num_requests: int, - request_id_prefix: str = "", - no_oversample: bool = False, - prefix_len: int = DEFAULT_PREFIX_LEN, - input_len: int = DEFAULT_INPUT_LEN, - output_len: int = DEFAULT_OUTPUT_LEN, - return_prompt_formatted: bool = False, - **kwargs, - ) -> list[SampleRequest]: - poem_lines = self.data - assert poem_lines is not None - # Calculate average token length for a poem line. - tokenized_lines = [tokenizer(line).input_ids for line in poem_lines] - avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines) - - # Build the base prompt. - base_prompt = "Pick as many lines as you can from these poem lines:\n" - base_msg = [{"role": "user", "content": base_prompt}] - base_fmt = tokenizer.apply_chat_template( - base_msg, # type: ignore[arg-type] - add_generation_prompt=True, - tokenize=False, - ) - assert isinstance(base_fmt, str) - base_offset = len(tokenizer(base_fmt).input_ids) - if input_len <= base_offset: - raise ValueError( - f"'input_len' must be higher than the base prompt length " - f"({base_offset})." - ) - - # Determine how many poem lines to use. - num_input_lines = max(round((input_len - base_offset) / avg_len), 1) - num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0) - prefix_lines = poem_lines[:num_prefix_lines] - - samples: list[SampleRequest] = [] - ind = 0 - while len(samples) < num_requests: - extra_lines = random.choices( - poem_lines, k=num_input_lines - num_prefix_lines - ) - prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}" - msg = [{"role": "user", "content": prompt}] - prompt_formatted = tokenizer.apply_chat_template( - msg, # type: ignore[arg-type] - add_generation_prompt=True, - tokenize=False, - ) - assert isinstance(prompt_formatted, str) - prompt_len = len(tokenizer(prompt_formatted).input_ids) - if prompt_len <= input_len: - samples.append( - SampleRequest( - prompt=( - prompt_formatted if return_prompt_formatted else prompt - ), - prompt_len=prompt_len, - expected_output_len=output_len, - request_id=request_id_prefix + str(ind), - ) - ) - ind += 1 - return samples - - # ----------------------------------------------------------------------------- # BurstGPT Dataset Implementation # ----------------------------------------------------------------------------- diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index b551361f55..9e3c0b57e4 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -1653,8 +1653,7 @@ def add_cli_args(parser: FlexibleArgumentParser): "--input-len", type=int, default=None, - help="General input length for datasets. Maps to dataset-specific " - "input length arguments (e.g., --random-input-len, --sonnet-input-len). " + help="General input length for datasets. Maps to --random-input-len. " "If not specified, uses dataset defaults.", ) parser.add_argument( @@ -1662,7 +1661,7 @@ def add_cli_args(parser: FlexibleArgumentParser): type=int, default=None, help="General output length for datasets. Maps to dataset-specific " - "output length arguments (e.g., --random-output-len, --sonnet-output-len). " + "output length arguments (e.g., --random-output-len, --hf-output-len). " "If not specified, uses dataset defaults.", ) parser.add_argument( @@ -2115,18 +2114,16 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]: raise ValueError( f"Cannot use '{args.dataset_name}' dataset with --dataset-path. " "Please specify the appropriate --dataset-name (e.g., " - "'sharegpt', 'custom', 'sonnet') for your dataset file: " + "'sharegpt', 'custom', 'hf') for your dataset file: " f"{args.dataset_path}" ) - # Map general --input-len and --output-len to all dataset-specific arguments + # Map general length options to dataset-specific arguments. if args.input_len is not None: args.random_input_len = args.input_len - args.sonnet_input_len = args.input_len if args.output_len is not None: args.random_output_len = args.output_len - args.sonnet_output_len = args.output_len args.sharegpt_output_len = args.output_len args.custom_output_len = args.output_len args.hf_output_len = args.output_len diff --git a/vllm/benchmarks/throughput.py b/vllm/benchmarks/throughput.py index d7c3c01454..6245b20e86 100644 --- a/vllm/benchmarks/throughput.py +++ b/vllm/benchmarks/throughput.py @@ -554,10 +554,6 @@ def _to_serve_args(args: argparse.Namespace) -> argparse.Namespace: # matching prior throughput behaviour of omitting output_len when unset. d["hf_output_len"] = output_len d["sharegpt_output_len"] = output_len - # sonnet reads dedicated attrs; fall back to SonnetDataset's own defaults. - d["sonnet_input_len"] = input_len if input_len is not None else 550 - d["sonnet_output_len"] = output_len if output_len is not None else 150 - d["sonnet_prefix_len"] = prefix_len # Explicit --enable-multimodal-chat wins; otherwise auto-enable for the # multimodal chat backend (preserves today's vllm-chat handling). Callers # without a --backend flag (e.g. bench mm-processor) drive every request @@ -710,14 +706,14 @@ def validate_args(args): ) # --prefix-len: only used when dataset_name is 'random', 'random-mm', - # 'sonnet', or not set. + # or not set. if ( - args.dataset_name not in {"random", "random-mm", "sonnet", None} + args.dataset_name not in {"random", "random-mm", None} and args.prefix_len is not None ): warnings.warn( "--prefix-len will be ignored since --dataset-name\ - is not 'random', 'random-mm', 'sonnet', or not set.", + is not 'random', 'random-mm', or not set.", stacklevel=2, ) @@ -803,7 +799,6 @@ def add_cli_args(parser: FlexibleArgumentParser): choices=[ "sharegpt", "random", - "sonnet", "burstgpt", "hf", "prefix_repetition",