[V1] Logprobs and prompt logprobs support (#9880)
This PR is adding support for sample logprobs & prompt logprobs to vLLM v1. New behavior: - During model execution, model runner computes sample logprobs (if user-provided logprobs setting is not None) and prompt logprobs (if user-provided prompt_logprobs setting is not None). For both sample and prompt logprobs, the engine core returns 3 vectors: token ids, token logprob values, token ranks. Ranks reflect tokens' 1-indexed positions in the vocabulary vector after sorting the vocabulary by log probability in descending order. - In scheduler.update_from_output(), sample and prompt logprobs are incorporated into the EngineCoreOutput data structure which is transferred to the engine client. If multiprocessing is enabled, then sample and prompt logprobs will be (de)serialized when the EngineCoreOutput data structure is (de)serialized. - During output processing, the LogprobsProcessor transforms the triplet of token ids, token logprobs values, and token ranks into the OpenAI-compatible List[Dict[token id,Logprob]] format (for sample and prompt logprobs respectively.) - Each Logprob instance (whether sample- or prompt-) consists of a token's log-probability, rank, and detokenized string representation. Note that logprob detokenization is handled by the LogprobsProcessor not the detokenizer. Signed-off-by: Andrew Feldman <afeldman@neuralmagic.com> Signed-off-by: Nick Hill <nhill@redhat.com> Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
co-authored by
rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Nick Hill
parent
538fab93cd
commit
0630d4537a
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.v1.engine.utils import (NUM_PROMPT_LOGPROBS_UNDER_TEST,
|
||||
NUM_SAMPLE_LOGPROBS_UNDER_TEST, PROMPT_LEN,
|
||||
TOKENIZER_NAME,
|
||||
DummyOutputProcessorTestVectors,
|
||||
generate_dummy_prompt_logprobs_tensors,
|
||||
generate_dummy_sample_logprobs)
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
|
||||
|
||||
from tests.v1.engine.utils import FULL_STRINGS # isort: skip
|
||||
|
||||
EngineCoreSampleLogprobsType = List[Tuple[torch.Tensor, torch.Tensor]]
|
||||
EngineCorePromptLogprobsType = Tuple[torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:
|
||||
"""Generate output processor dummy test vectors, without logprobs
|
||||
|
||||
Returns:
|
||||
DummyOutputProcessorTestVectors instance with no logprobs
|
||||
"""
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
|
||||
vllm_config = EngineArgs(model=TOKENIZER_NAME).create_engine_config()
|
||||
# Tokenize prompts under test & create dummy generated tokens
|
||||
prompt_tokens = [
|
||||
tokenizer(text).input_ids[:PROMPT_LEN] for text in FULL_STRINGS
|
||||
]
|
||||
generation_tokens = [
|
||||
tokenizer(text).input_ids[PROMPT_LEN:] for text in FULL_STRINGS
|
||||
]
|
||||
# Generate prompt strings
|
||||
prompt_strings = [
|
||||
tokenizer.decode(prompt_tokens, skip_special_tokens=True)
|
||||
for prompt_tokens in prompt_tokens
|
||||
]
|
||||
prompt_strings_len = [
|
||||
len(prompt_string) for prompt_string in prompt_strings
|
||||
]
|
||||
return DummyOutputProcessorTestVectors(
|
||||
tokenizer=tokenizer,
|
||||
tokenizer_group=init_tokenizer_from_configs(
|
||||
vllm_config.model_config, vllm_config.scheduler_config,
|
||||
vllm_config.parallel_config, vllm_config.lora_config),
|
||||
vllm_config=vllm_config,
|
||||
full_tokens=[tokenizer(text).input_ids for text in FULL_STRINGS],
|
||||
prompt_tokens=prompt_tokens,
|
||||
generation_tokens=generation_tokens,
|
||||
prompt_strings=prompt_strings,
|
||||
prompt_strings_len=prompt_strings_len,
|
||||
generation_strings=[
|
||||
text[prompt_len:]
|
||||
for text, prompt_len in zip(FULL_STRINGS, prompt_strings_len)
|
||||
],
|
||||
prompt_logprobs=[],
|
||||
generation_logprobs=[])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_test_vectors() -> DummyOutputProcessorTestVectors:
|
||||
"""Generate output processor dummy test vectors, with logprobs
|
||||
|
||||
Returns:
|
||||
DummyOutputProcessorTestVectors instance with logprobs
|
||||
"""
|
||||
# Build dummy test vectors without logprobs
|
||||
dtv = _build_test_vectors_no_logprobs()
|
||||
# Inject logprobs into dummy test vectors
|
||||
# data structure
|
||||
dtv.generation_logprobs = [
|
||||
generate_dummy_sample_logprobs(
|
||||
sampled_tokens_list=tokens_list,
|
||||
num_logprobs=NUM_SAMPLE_LOGPROBS_UNDER_TEST,
|
||||
tokenizer=dtv.tokenizer) for tokens_list in dtv.generation_tokens
|
||||
]
|
||||
dtv.prompt_logprobs = [
|
||||
generate_dummy_prompt_logprobs_tensors(
|
||||
prompt_tokens_list=tokens_list,
|
||||
num_logprobs=NUM_PROMPT_LOGPROBS_UNDER_TEST,
|
||||
tokenizer=dtv.tokenizer) for tokens_list in dtv.prompt_tokens
|
||||
]
|
||||
return dtv
|
||||
@@ -2,10 +2,11 @@
|
||||
|
||||
import asyncio
|
||||
from contextlib import ExitStack
|
||||
from typing import List, Tuple
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.v1.engine.utils import PLP_APC_UNSUPPORTED_MSG
|
||||
from vllm import SamplingParams
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
@@ -21,13 +22,19 @@ ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B",
|
||||
disable_log_requests=True)
|
||||
|
||||
|
||||
async def generate(engine: AsyncLLM, request_id: str,
|
||||
async def generate(engine: AsyncLLM,
|
||||
request_id: str,
|
||||
output_kind: RequestOutputKind,
|
||||
max_tokens: int) -> Tuple[int, str]:
|
||||
max_tokens: int,
|
||||
prompt_logprobs: Optional[int] = None) -> Tuple[int, str]:
|
||||
# Ensure generate doesn't complete too fast for cancellation test.
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
count = 0
|
||||
sampling_params = SamplingParams(max_tokens=max_tokens,
|
||||
output_kind=output_kind,
|
||||
temperature=0)
|
||||
temperature=0,
|
||||
prompt_logprobs=prompt_logprobs)
|
||||
async for out in engine.generate(request_id=request_id,
|
||||
prompt="Hello my name is Robert and",
|
||||
sampling_params=sampling_params):
|
||||
@@ -43,6 +50,40 @@ async def generate(engine: AsyncLLM, request_id: str,
|
||||
return count, request_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_llm_refuses_prompt_logprobs_with_apc(
|
||||
monkeypatch, output_kind: RequestOutputKind):
|
||||
"""Test passes if AsyncLLM raises an exception when it is configured
|
||||
for automatic prefix caching and it receives a request with
|
||||
prompt_logprobs enabled, which is incompatible."""
|
||||
# TODO(rickyx): Remove monkeypatch VLLM_USE_V1 setting once we have a
|
||||
# better way to test V1 so that in the future when we switch, we don't
|
||||
# have to change all the tests.
|
||||
monkeypatch.setenv("VLLM_USE_V1", "1")
|
||||
# Create AsyncLLM engine with APC
|
||||
apc_engine_args = AsyncEngineArgs(model="facebook/opt-125m",
|
||||
enable_prefix_caching=True,
|
||||
gpu_memory_utilization=0.8,
|
||||
disable_log_requests=True)
|
||||
engine = AsyncLLM.from_engine_args(apc_engine_args)
|
||||
try:
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
# Issue a request with prompt logprobs enabled, which should fail
|
||||
await asyncio.create_task(
|
||||
generate(engine,
|
||||
"request-0",
|
||||
output_kind,
|
||||
10,
|
||||
prompt_logprobs=5))
|
||||
# Validate exception string is correct
|
||||
assert str(excinfo.value) == PLP_APC_UNSUPPORTED_MSG
|
||||
finally:
|
||||
# Shut down engine
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.v1.engine.utils import PLP_APC_UNSUPPORTED_MSG
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
|
||||
def test_llm_engine_refuses_prompt_logprobs_with_apc(monkeypatch):
|
||||
"""Test passes if LLMEngine raises an exception when it is configured
|
||||
for automatic prefix caching and it receives a request with
|
||||
prompt_logprobs enabled, which is incompatible."""
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V1", "1")
|
||||
# TODO(nick): Single-proc to work around a ZMQ shutdown hang for now.
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
LLM(model="facebook/opt-125m", enable_prefix_caching=True).generate(
|
||||
"Hello, my name is",
|
||||
SamplingParams(temperature=0.8, top_p=0.95, prompt_logprobs=5))
|
||||
|
||||
# Validate exception string is correct
|
||||
assert str(excinfo.value) == PLP_APC_UNSUPPORTED_MSG
|
||||
@@ -1,82 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import List
|
||||
import math
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from tests.v1.engine.utils import (NUM_PROMPT_LOGPROBS_UNDER_TEST,
|
||||
NUM_SAMPLE_LOGPROBS_UNDER_TEST,
|
||||
STOP_STRINGS,
|
||||
DummyOutputProcessorTestVectors,
|
||||
MockEngineCore)
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
|
||||
from vllm.sequence import PromptLogprobs, SampleLogprobs
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
|
||||
TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
|
||||
VLLM_CONFIG = EngineArgs(model=TOKENIZER_NAME).create_engine_config()
|
||||
TOKENIZER_GROUP = init_tokenizer_from_configs(VLLM_CONFIG.model_config,
|
||||
VLLM_CONFIG.scheduler_config,
|
||||
VLLM_CONFIG.parallel_config,
|
||||
VLLM_CONFIG.lora_config)
|
||||
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
|
||||
|
||||
FULL_STRINGS = [
|
||||
"My name is Robert from Neural Magic and I love working on vLLM so much!",
|
||||
"Red Hat is the best open source company by far across Linux, K8s, and AI.",
|
||||
"Nick is the name of my brother in addition to my colleague from Red Hat.",
|
||||
]
|
||||
def _ref_convert_id_to_token(
|
||||
tokenizer: AnyTokenizer,
|
||||
token_id: int,
|
||||
) -> str:
|
||||
"""Reference impl of logprobs detokenization.
|
||||
|
||||
STOP_STRINGS = ["I love working on", "company by far", "brother in"]
|
||||
Args:
|
||||
tokenizer: tokenizer used by the model under test
|
||||
token_id: convert this token id
|
||||
|
||||
FULL_TOKENS = [tokenizer(text).input_ids for text in FULL_STRINGS]
|
||||
PROMPT_LEN = 5
|
||||
PROMPT_TOKENS = [
|
||||
tokenizer(text).input_ids[:PROMPT_LEN] for text in FULL_STRINGS
|
||||
]
|
||||
GENERATION_TOKENS = [
|
||||
tokenizer(text).input_ids[PROMPT_LEN:] for text in FULL_STRINGS
|
||||
]
|
||||
PROMPT_STRINGS = [
|
||||
tokenizer.decode(prompt_tokens, skip_special_tokens=True)
|
||||
for prompt_tokens in PROMPT_TOKENS
|
||||
]
|
||||
PROMPT_STRINGS_LEN = [len(prompt_string) for prompt_string in PROMPT_STRINGS]
|
||||
GENERATION_STRINGS = [
|
||||
text[prompt_len:]
|
||||
for text, prompt_len in zip(FULL_STRINGS, PROMPT_STRINGS_LEN)
|
||||
]
|
||||
|
||||
|
||||
class MockEngineCore:
|
||||
"""Mock outputs form premade tokens lists."""
|
||||
|
||||
def __init__(self, tokens_list: List[List[int]]):
|
||||
self.tokens_list = tokens_list
|
||||
self.current_idx = 0
|
||||
|
||||
def get_outputs(self) -> List[EngineCoreOutput]:
|
||||
token_idx = self.current_idx
|
||||
self.current_idx += 1
|
||||
|
||||
outputs = []
|
||||
for req_idx, token_ids in enumerate(self.tokens_list):
|
||||
if len(token_ids) > token_idx:
|
||||
output = EngineCoreOutput(request_id=f"request-{req_idx}",
|
||||
new_token_ids=[token_ids[token_idx]],
|
||||
finished=False)
|
||||
if token_idx == len(token_ids) - 1:
|
||||
output.finished = True
|
||||
output.finish_reason = "stopped"
|
||||
outputs.append(output)
|
||||
|
||||
return outputs
|
||||
Returns:
|
||||
String representation of input token id
|
||||
"""
|
||||
return tokenizer.convert_ids_to_tokens(token_id) or ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_output_kind",
|
||||
[RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
|
||||
def test_incremental_detokenization(request_output_kind: RequestOutputKind):
|
||||
output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
|
||||
engine_core = MockEngineCore(GENERATION_TOKENS)
|
||||
def test_incremental_detokenization(request_output_kind: RequestOutputKind,
|
||||
dummy_test_vectors):
|
||||
output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
|
||||
log_stats=False)
|
||||
engine_core = MockEngineCore(
|
||||
tokens_list=dummy_test_vectors.generation_tokens)
|
||||
|
||||
# Make N requests.
|
||||
requests = [
|
||||
@@ -94,10 +59,10 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
|
||||
spaces_between_special_tokens=False,
|
||||
output_kind=request_output_kind,
|
||||
stop=[],
|
||||
include_stop_str_in_output=False))
|
||||
for idx, (
|
||||
prompt,
|
||||
prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
|
||||
include_stop_str_in_output=False,
|
||||
)) for idx, (prompt, prompt_tokens) in enumerate(
|
||||
zip(dummy_test_vectors.prompt_strings,
|
||||
dummy_test_vectors.prompt_tokens))
|
||||
]
|
||||
|
||||
# Add requests to the detokenizer.
|
||||
@@ -113,7 +78,7 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
|
||||
break
|
||||
|
||||
# Step the Detokenizer.
|
||||
processed_outputs = output_processor.process_outputs(outputs, )
|
||||
processed_outputs = output_processor.process_outputs(outputs)
|
||||
request_outputs = processed_outputs.request_outputs
|
||||
requests_to_abort = processed_outputs.reqs_to_abort
|
||||
assert len(requests_to_abort) == 0
|
||||
@@ -132,7 +97,8 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
|
||||
|
||||
# Confirmed tracked values matches what we expected.
|
||||
for idx, (ref_gen_str, ref_gen_toks) in enumerate(
|
||||
zip(GENERATION_STRINGS, GENERATION_TOKENS)):
|
||||
zip(dummy_test_vectors.generation_strings,
|
||||
dummy_test_vectors.generation_tokens)):
|
||||
gen_str = gen_strings[f"request-{idx}"]
|
||||
gen_toks = gen_tokens[f"request-{idx}"]
|
||||
|
||||
@@ -143,15 +109,390 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("include_stop_str_in_output", [True, False])
|
||||
def test_stop_string(include_stop_str_in_output: bool):
|
||||
output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
|
||||
engine_core = MockEngineCore(GENERATION_TOKENS)
|
||||
def _validate_logprobs(
|
||||
gen_tokens: Dict[str, List[int]],
|
||||
gen_logprobs: Dict[str, Optional[SampleLogprobs]],
|
||||
gen_prompt_logprobs: Dict[str, Optional[PromptLogprobs]],
|
||||
gen_cumulative_logprob: Dict[str, float],
|
||||
dtv: DummyOutputProcessorTestVectors,
|
||||
request_id_list: List[str],
|
||||
num_sample_logprobs: Optional[int],
|
||||
num_prompt_logprobs: Optional[int],
|
||||
) -> None:
|
||||
for req_idx, req_id in enumerate(request_id_list):
|
||||
new_tokens = gen_tokens[req_id]
|
||||
logprobs = gen_logprobs[req_id]
|
||||
prompt_logprobs = gen_prompt_logprobs[req_id]
|
||||
cumulative_logprob = gen_cumulative_logprob[req_id]
|
||||
prompt_token_ids = dtv.prompt_tokens[req_idx]
|
||||
ref_logprobs = dtv.generation_logprobs[req_idx]
|
||||
ref_prompt_logprobs = dtv.prompt_logprobs[req_idx]
|
||||
if num_sample_logprobs is not None:
|
||||
# Validate sample logprobs
|
||||
assert logprobs is not None, (f"Request {req_id} requires sample"
|
||||
" logprobs but sample logprobs are"
|
||||
" None.")
|
||||
# Require num sampled tokens to match num
|
||||
# sampled logprobs - especially important
|
||||
# to check since the detokenizer can cause
|
||||
# a request to finish early due to a stop
|
||||
# string being hit
|
||||
num_new_tokens = len(new_tokens)
|
||||
len_sample_logprobs = len(logprobs)
|
||||
assert num_new_tokens == len_sample_logprobs, (
|
||||
f"Request {req_id} has {num_new_tokens}"
|
||||
" completion tokens but has"
|
||||
f" {len_sample_logprobs} sample logprobs.")
|
||||
ref_cumulative_logprob = 0.0
|
||||
for idx, (sampled_token,
|
||||
pos_logprob_dict) in enumerate(zip(new_tokens,
|
||||
logprobs)):
|
||||
# Break out the reference log probability value &
|
||||
# logprob token id tensors associated with this
|
||||
# position in the completion. Also break out the
|
||||
# sampled token ranks
|
||||
(ref_pos_logprob_toks, ref_pos_logprob_vals,
|
||||
ref_sampled_token_rank) = ref_logprobs[idx]
|
||||
# For each position in the completion sequence,
|
||||
# ensure the actual sampled token is among the
|
||||
# logprobs
|
||||
assert sampled_token in pos_logprob_dict, (
|
||||
f"Sampled token {sampled_token} not"
|
||||
f" present in logprob at index {idx}")
|
||||
|
||||
# Validate number of sample logprobs
|
||||
num_lp_toks = len(pos_logprob_dict)
|
||||
assert (num_lp_toks == num_sample_logprobs
|
||||
or num_lp_toks == num_sample_logprobs +
|
||||
1), ("Valid numbers of sample logprobs are"
|
||||
f" {num_sample_logprobs} or"
|
||||
f" {num_sample_logprobs+1} but"
|
||||
f" {num_lp_toks} logprobs found at"
|
||||
f" position {idx}. Logprobs dict:"
|
||||
f" {pos_logprob_dict}")
|
||||
|
||||
# Validate sampled token logprob rank
|
||||
smp_lp = pos_logprob_dict[sampled_token]
|
||||
smp_lp_rank = smp_lp.rank
|
||||
assert (ref_sampled_token_rank == smp_lp_rank), (
|
||||
"Sampled token logprob rank"
|
||||
f" {smp_lp_rank} does not match"
|
||||
" correct value"
|
||||
f" {ref_sampled_token_rank}"
|
||||
f" in Logprob {smp_lp}")
|
||||
|
||||
# Validate that the logprob processor yields
|
||||
# the correct log probabilities and valid
|
||||
# rankings
|
||||
rank_one_appears = False
|
||||
for jdx in range(1, len(ref_pos_logprob_toks)):
|
||||
# Iterate over the (logprob val,logprob tok id)
|
||||
# pairs expected by the test fixture at this
|
||||
# position in the completion.
|
||||
ref_lp_val = ref_pos_logprob_vals[jdx]
|
||||
ref_tok_id = ref_pos_logprob_toks[jdx]
|
||||
assert ref_tok_id in pos_logprob_dict, (
|
||||
f"Expected token {ref_tok_id} to be"
|
||||
f" in logprob dict but it is not.")
|
||||
|
||||
# Extract actually-generated logprob
|
||||
# info
|
||||
lp = pos_logprob_dict[ref_tok_id]
|
||||
lp_val = lp.logprob
|
||||
lp_rank = lp.rank
|
||||
|
||||
# A "top" (rank 1) logprob must be
|
||||
# present
|
||||
rank_one_appears = (True
|
||||
if lp_rank == 1 else rank_one_appears)
|
||||
|
||||
# Rank must be >= 1
|
||||
assert lp_rank >= 1, (f"Logprob {lp} has invalid"
|
||||
f" rank {lp_rank} < 1."
|
||||
f" Logprob dict: {pos_logprob_dict}")
|
||||
|
||||
# Validate log probability
|
||||
assert math.isclose(lp_val, ref_lp_val), (
|
||||
f"Token id {ref_tok_id} appears in logprobs dict"
|
||||
f" at position {idx} in completion with log"
|
||||
f" probability {lp_val} but {ref_lp_val} was"
|
||||
f" expected. Logprob: {lp}")
|
||||
|
||||
assert rank_one_appears, (f"No Logprob has rank 1"
|
||||
" in the following Logprob"
|
||||
f" dict: {pos_logprob_dict}")
|
||||
|
||||
# Validate logprobs detokenization
|
||||
for lp_tok in pos_logprob_dict:
|
||||
# Confirm that sample logprob decoded token matches
|
||||
# the logprob token id at this sequence position
|
||||
decoded_token = pos_logprob_dict[lp_tok].decoded_token
|
||||
ref_decoded_token = _ref_convert_id_to_token(
|
||||
dtv.tokenizer, lp_tok)
|
||||
assert decoded_token == ref_decoded_token, (
|
||||
f"Sampled logprob token id {lp_tok} decodes to"
|
||||
f" {ref_decoded_token} but Logprob decoded"
|
||||
f" token is {decoded_token} instead"
|
||||
f" (at position {idx})")
|
||||
|
||||
ref_cumulative_logprob += pos_logprob_dict[
|
||||
sampled_token].logprob
|
||||
# Assert that cumulative logprobs are correct
|
||||
assert math.isclose(cumulative_logprob, ref_cumulative_logprob)
|
||||
else:
|
||||
# Sample logprobs disabled for this request
|
||||
assert logprobs is None
|
||||
assert cumulative_logprob is None
|
||||
|
||||
if num_prompt_logprobs is not None:
|
||||
# Validate prompt logprobs
|
||||
assert prompt_logprobs is not None, (
|
||||
f"Request {req_id} requires prompt"
|
||||
" logprobs but prompt logprobs are"
|
||||
" None.")
|
||||
# Require num prompt tokens to match num
|
||||
# prompt logprobs
|
||||
num_prompt_tokens = len(prompt_token_ids)
|
||||
len_prompt_logprobs = len(prompt_logprobs)
|
||||
assert num_prompt_tokens == len_prompt_logprobs, (
|
||||
f"Request {req_id} has {num_prompt_tokens}"
|
||||
" prompt tokens but has"
|
||||
f" {len_prompt_logprobs} prompt logprobs.")
|
||||
# First prompt logprob is None
|
||||
first_plp_dict = prompt_logprobs[0]
|
||||
assert first_plp_dict is None, (
|
||||
f"Request {req_id} first prompt logprob"
|
||||
f" should be None but has following value"
|
||||
f" instead: {first_plp_dict}")
|
||||
# Break out the reference prompt log prob value &
|
||||
# logprob token id matrices for the whole prompt.
|
||||
# Also break out the prompt token rank vector
|
||||
(ref_prompt_logprob_toks, ref_prompt_logprob_vals,
|
||||
ref_prompt_token_ranks) = ref_prompt_logprobs
|
||||
for idx, (prompt_token, pos_logprob_dict) in enumerate(
|
||||
zip(prompt_token_ids[1:], prompt_logprobs[1:])):
|
||||
|
||||
# Break out the reference prompt log prob value
|
||||
# vector, prompt logprob token id vector, and
|
||||
# prompt token rank at the current position.
|
||||
(ref_pos_prompt_logprob_toks, ref_pos_prompt_logprob_vals,
|
||||
ref_pos_prompt_token_rank) = (ref_prompt_logprob_toks[idx, :],
|
||||
ref_prompt_logprob_vals[idx, :],
|
||||
ref_prompt_token_ranks[idx])
|
||||
|
||||
# For each position in the prompt sequence,
|
||||
# ensure the actual prompt token is among the
|
||||
# logprobs
|
||||
assert prompt_token in pos_logprob_dict, (
|
||||
f"Prompt token {prompt_token} not"
|
||||
f" present in logprob at index {idx}")
|
||||
# Validate number of prompt logprobs
|
||||
num_plp_toks = len(pos_logprob_dict)
|
||||
assert (num_plp_toks == num_prompt_logprobs
|
||||
or num_plp_toks == num_prompt_logprobs +
|
||||
1), ("Valid numbers of prompt logprobs are"
|
||||
f" {num_prompt_logprobs} or"
|
||||
f" {num_prompt_logprobs+1} but"
|
||||
f" {num_plp_toks} logprobs found at"
|
||||
f" position {idx}. Logprobs dict:"
|
||||
f" {pos_logprob_dict}")
|
||||
|
||||
# Validate prompt token logprob rank
|
||||
prmpt_tok_lp = pos_logprob_dict[prompt_token]
|
||||
prmpt_tok_lp_rank = prmpt_tok_lp.rank
|
||||
ref_prmpt_tok_lp_rank = ref_pos_prompt_token_rank
|
||||
assert (ref_prmpt_tok_lp_rank == prmpt_tok_lp_rank), (
|
||||
"Prompt token logprob rank"
|
||||
f" {prmpt_tok_lp_rank} does not match"
|
||||
" correct value"
|
||||
f" {ref_prmpt_tok_lp_rank}"
|
||||
f" in Logprob {prmpt_tok_lp}")
|
||||
|
||||
# Validate that the logprob processor yields
|
||||
# the correct prompt log probs and valid
|
||||
# rankings
|
||||
rank_one_appears = False
|
||||
for jdx in range(1, len(ref_pos_prompt_logprob_toks)):
|
||||
# Iterate over the (logprob val,logprob tok id)
|
||||
# pairs expected by the test fixture at this
|
||||
# position in the completion.
|
||||
ref_plp_val = float(ref_pos_prompt_logprob_vals[jdx])
|
||||
ref_tok_id = int(ref_pos_prompt_logprob_toks[jdx])
|
||||
assert ref_tok_id in pos_logprob_dict, (
|
||||
f"Expected token {ref_tok_id} to be"
|
||||
f" in logprob dict but it is not.")
|
||||
|
||||
# Extract actually-generated logprob
|
||||
# info
|
||||
plp = pos_logprob_dict[ref_tok_id]
|
||||
plp_val = plp.logprob
|
||||
plp_rank = plp.rank
|
||||
|
||||
# A "top" (rank 1) logprob must be
|
||||
# present
|
||||
rank_one_appears = (True
|
||||
if plp_rank == 1 else rank_one_appears)
|
||||
|
||||
# Rank must be >= 1
|
||||
assert plp_rank >= 1, (
|
||||
f"Logprob {plp} has invalid"
|
||||
f" rank {plp_rank} < 1."
|
||||
f" Logprob dict: {pos_logprob_dict}")
|
||||
|
||||
# Validate log probability
|
||||
assert math.isclose(plp_val, ref_plp_val), (
|
||||
f"Token id {ref_tok_id} appears in logprobs dict"
|
||||
f" at position {idx} in completion with log"
|
||||
f" probability {plp_val} but {ref_plp_val} was"
|
||||
f" expected. Logprob: {plp}")
|
||||
|
||||
assert rank_one_appears, (f"No Logprob has rank 1"
|
||||
" in the following Logprob"
|
||||
f" dict: {pos_logprob_dict}")
|
||||
|
||||
# Validate prompt logprob detokenization
|
||||
for plp_tok in pos_logprob_dict:
|
||||
# Confirm that prompt logprob decoded token matches
|
||||
# the logprob token id at this sequence position
|
||||
decoded_token = pos_logprob_dict[plp_tok].decoded_token
|
||||
ref_decoded_token = _ref_convert_id_to_token(
|
||||
dtv.tokenizer, plp_tok)
|
||||
assert decoded_token == ref_decoded_token, (
|
||||
f"Prompt logprob token id {plp_tok} decodes to"
|
||||
f" {ref_decoded_token} but Logprob decoded"
|
||||
f" token is {decoded_token} instead"
|
||||
f" (at position {idx})")
|
||||
else:
|
||||
# Prompt logprobs disabled for this request
|
||||
assert prompt_logprobs is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_output_kind",
|
||||
[RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
|
||||
@pytest.mark.parametrize("num_sample_logprobs",
|
||||
[None, NUM_SAMPLE_LOGPROBS_UNDER_TEST])
|
||||
@pytest.mark.parametrize("num_prompt_logprobs",
|
||||
[None, NUM_PROMPT_LOGPROBS_UNDER_TEST])
|
||||
def test_logprobs_processor(request_output_kind: RequestOutputKind,
|
||||
num_sample_logprobs: Optional[int],
|
||||
num_prompt_logprobs: Optional[int],
|
||||
dummy_test_vectors):
|
||||
output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
|
||||
log_stats=False)
|
||||
engine_core = MockEngineCore(
|
||||
tokens_list=dummy_test_vectors.generation_tokens,
|
||||
generated_logprobs_raw=None if num_sample_logprobs is None else
|
||||
dummy_test_vectors.generation_logprobs,
|
||||
prompt_logprobs_raw=None
|
||||
if num_prompt_logprobs is None else dummy_test_vectors.prompt_logprobs)
|
||||
|
||||
# Make N requests.
|
||||
request_id_list = [
|
||||
f"request-{idx}"
|
||||
for idx in range(len(dummy_test_vectors.prompt_strings))
|
||||
]
|
||||
requests = [
|
||||
EngineCoreRequest(request_id=request_id_list[idx],
|
||||
prompt=prompt,
|
||||
prompt_token_ids=prompt_tokens,
|
||||
arrival_time=0,
|
||||
mm_inputs=None,
|
||||
mm_hashes=None,
|
||||
mm_placeholders=None,
|
||||
eos_token_id=None,
|
||||
lora_request=None,
|
||||
sampling_params=SamplingParams(
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=False,
|
||||
output_kind=request_output_kind,
|
||||
stop=[],
|
||||
include_stop_str_in_output=False,
|
||||
logprobs=num_sample_logprobs,
|
||||
prompt_logprobs=num_prompt_logprobs,
|
||||
)) for idx, (prompt, prompt_tokens) in enumerate(
|
||||
zip(dummy_test_vectors.prompt_strings,
|
||||
dummy_test_vectors.prompt_tokens))
|
||||
]
|
||||
|
||||
# Add requests to the detokenizer.
|
||||
for request in requests:
|
||||
output_processor.add_request(request)
|
||||
|
||||
gen_tokens = {}
|
||||
gen_logprobs = {}
|
||||
gen_prompt_logprobs = {}
|
||||
gen_cumulative_logprobs = {}
|
||||
while True:
|
||||
# Mock output from the EngineCore.
|
||||
outputs = engine_core.get_outputs()
|
||||
if len(outputs) == 0:
|
||||
break
|
||||
|
||||
# Step the logprobs processor.
|
||||
processed_outputs = output_processor.process_outputs(outputs)
|
||||
request_outputs = processed_outputs.request_outputs
|
||||
requests_to_abort = processed_outputs.reqs_to_abort
|
||||
assert len(requests_to_abort) == 0
|
||||
|
||||
# Update tracking.
|
||||
for request_output in request_outputs:
|
||||
request_id = request_output.request_id
|
||||
new_tokens = request_output.outputs[0].token_ids
|
||||
prompt_logprobs = request_output.prompt_logprobs
|
||||
logprobs = request_output.outputs[0].logprobs
|
||||
gen_cumulative_logprobs[request_id] = request_output.outputs[
|
||||
0].cumulative_logprob
|
||||
if request_id not in gen_logprobs:
|
||||
# Start tracking sample and prompt logprobs for this request
|
||||
gen_tokens[request_id] = new_tokens
|
||||
gen_logprobs[request_id] = logprobs
|
||||
gen_prompt_logprobs[request_id] = prompt_logprobs
|
||||
else:
|
||||
# Extend logprobs tracker
|
||||
gen_tokens[request_id].extend(new_tokens)
|
||||
lp = gen_logprobs[request_id]
|
||||
plp = gen_prompt_logprobs[request_id]
|
||||
if lp:
|
||||
lp.extend(logprobs)
|
||||
if plp:
|
||||
plp.extend(prompt_logprobs)
|
||||
|
||||
# Confirmed tracked logprobs match what we expect
|
||||
_validate_logprobs(gen_tokens, gen_logprobs, gen_prompt_logprobs,
|
||||
gen_cumulative_logprobs, dummy_test_vectors,
|
||||
request_id_list, num_sample_logprobs,
|
||||
num_prompt_logprobs)
|
||||
|
||||
assert output_processor.get_num_unfinished_requests() == 0
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("include_stop_str_in_output", [True, False])
|
||||
@pytest.mark.parametrize("num_sample_logprobs",
|
||||
[None, NUM_SAMPLE_LOGPROBS_UNDER_TEST])
|
||||
@pytest.mark.parametrize("num_prompt_logprobs",
|
||||
[None, NUM_PROMPT_LOGPROBS_UNDER_TEST])
|
||||
def test_stop_string(include_stop_str_in_output: bool,
|
||||
num_sample_logprobs: Optional[int],
|
||||
num_prompt_logprobs: Optional[int], dummy_test_vectors):
|
||||
output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
|
||||
log_stats=False)
|
||||
engine_core = MockEngineCore(
|
||||
tokens_list=dummy_test_vectors.generation_tokens,
|
||||
generated_logprobs_raw=dummy_test_vectors.generation_logprobs
|
||||
if num_sample_logprobs else None,
|
||||
prompt_logprobs_raw=dummy_test_vectors.prompt_logprobs
|
||||
if num_prompt_logprobs else None)
|
||||
|
||||
# Make N requests.
|
||||
request_id_list = [
|
||||
f"request-{idx}"
|
||||
for idx in range(len(dummy_test_vectors.prompt_strings))
|
||||
]
|
||||
requests = [
|
||||
EngineCoreRequest(
|
||||
request_id=f"request-{idx}",
|
||||
request_id=request_id_list[idx],
|
||||
prompt=prompt,
|
||||
prompt_token_ids=prompt_tokens,
|
||||
arrival_time=0,
|
||||
@@ -166,9 +507,11 @@ def test_stop_string(include_stop_str_in_output: bool):
|
||||
output_kind=RequestOutputKind.DELTA,
|
||||
stop=STOP_STRINGS,
|
||||
include_stop_str_in_output=include_stop_str_in_output,
|
||||
)) for idx, (
|
||||
prompt,
|
||||
prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
|
||||
logprobs=num_sample_logprobs,
|
||||
prompt_logprobs=num_prompt_logprobs,
|
||||
)) for idx, (prompt, prompt_tokens) in enumerate(
|
||||
zip(dummy_test_vectors.prompt_strings,
|
||||
dummy_test_vectors.prompt_tokens))
|
||||
]
|
||||
|
||||
# Add requests to the detokenizer.
|
||||
@@ -176,6 +519,10 @@ def test_stop_string(include_stop_str_in_output: bool):
|
||||
output_processor.add_request(request)
|
||||
|
||||
gen_strings = {}
|
||||
gen_tokens = {}
|
||||
gen_logprobs = {}
|
||||
gen_prompt_logprobs = {}
|
||||
gen_cumulative_logprobs = {}
|
||||
aborted = []
|
||||
while True:
|
||||
# Mock output from the EngineCore.
|
||||
@@ -199,14 +546,29 @@ def test_stop_string(include_stop_str_in_output: bool):
|
||||
|
||||
request_id = request_output.request_id
|
||||
new_text = request_output.outputs[0].text
|
||||
new_tokens = request_output.outputs[0].token_ids
|
||||
prompt_logprobs = request_output.prompt_logprobs
|
||||
logprobs = request_output.outputs[0].logprobs
|
||||
gen_cumulative_logprobs[request_id] = request_output.outputs[
|
||||
0].cumulative_logprob
|
||||
if request_id not in gen_strings:
|
||||
gen_strings[request_id] = new_text
|
||||
gen_tokens[request_id] = new_tokens
|
||||
gen_logprobs[request_id] = logprobs
|
||||
gen_prompt_logprobs[request_id] = prompt_logprobs
|
||||
else:
|
||||
gen_strings[request_id] += new_text
|
||||
gen_tokens[request_id].extend(new_tokens)
|
||||
lp = gen_logprobs[request_id]
|
||||
plp = gen_prompt_logprobs[request_id]
|
||||
if lp:
|
||||
lp.extend(logprobs)
|
||||
if plp:
|
||||
plp.extend(prompt_logprobs)
|
||||
|
||||
# Confirmed tracked values matches what we expected.
|
||||
for idx, (ref_gen_str,
|
||||
stop_str) in enumerate(zip(GENERATION_STRINGS, STOP_STRINGS)):
|
||||
for idx, (ref_gen_str, stop_str) in enumerate(
|
||||
zip(dummy_test_vectors.generation_strings, STOP_STRINGS)):
|
||||
|
||||
# Request should be aborted.
|
||||
request_id = f"request-{idx}"
|
||||
@@ -227,13 +589,20 @@ def test_stop_string(include_stop_str_in_output: bool):
|
||||
assert gen_str == ref_str_exc_stop, (
|
||||
f"{gen_str=}, {ref_str_exc_stop=}")
|
||||
|
||||
# Confirmed tracked logprobs match what we expect
|
||||
_validate_logprobs(gen_tokens, gen_logprobs, gen_prompt_logprobs,
|
||||
gen_cumulative_logprobs, dummy_test_vectors,
|
||||
request_id_list, num_sample_logprobs,
|
||||
num_prompt_logprobs)
|
||||
|
||||
assert output_processor.get_num_unfinished_requests() == 0
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
|
||||
def test_iteration_stats():
|
||||
output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=True)
|
||||
engine_core = MockEngineCore(GENERATION_TOKENS)
|
||||
def test_iteration_stats(dummy_test_vectors):
|
||||
output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
|
||||
log_stats=True)
|
||||
engine_core = MockEngineCore(dummy_test_vectors.generation_tokens)
|
||||
|
||||
# Make N requests.
|
||||
requests = [
|
||||
@@ -248,13 +617,13 @@ def test_iteration_stats():
|
||||
eos_token_id=None,
|
||||
lora_request=None,
|
||||
sampling_params=SamplingParams(),
|
||||
) for idx, (
|
||||
prompt,
|
||||
prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
|
||||
) for idx, (prompt, prompt_tokens) in enumerate(
|
||||
zip(dummy_test_vectors.prompt_strings,
|
||||
dummy_test_vectors.prompt_tokens))
|
||||
]
|
||||
|
||||
# Add all requests except one to the OutputProcessor.
|
||||
num_active = len(GENERATION_TOKENS) - 1
|
||||
num_active = len(dummy_test_vectors.generation_tokens) - 1
|
||||
for request in requests[:num_active]:
|
||||
output_processor.add_request(request)
|
||||
inactive_request = requests[num_active]
|
||||
@@ -263,8 +632,10 @@ def test_iteration_stats():
|
||||
outputs = engine_core.get_outputs()[:num_active]
|
||||
processed_outputs = output_processor.process_outputs(outputs)
|
||||
iteration_stats = processed_outputs.iteration_stats
|
||||
total_prompt_tokens = sum(
|
||||
[len(prompt_tokens) for prompt_tokens in PROMPT_TOKENS[:num_active]])
|
||||
total_prompt_tokens = sum([
|
||||
len(prompt_tokens)
|
||||
for prompt_tokens in dummy_test_vectors.prompt_tokens[:num_active]
|
||||
])
|
||||
|
||||
assert iteration_stats.num_prompt_tokens == total_prompt_tokens
|
||||
assert iteration_stats.num_generation_tokens == num_active
|
||||
@@ -283,7 +654,7 @@ def test_iteration_stats():
|
||||
outputs = engine_core.get_outputs()[:num_active]
|
||||
processed_outputs = output_processor.process_outputs(outputs)
|
||||
iteration_stats = processed_outputs.iteration_stats
|
||||
total_prompt_tokens = len(PROMPT_TOKENS[num_active - 1])
|
||||
total_prompt_tokens = len(dummy_test_vectors.prompt_tokens[num_active - 1])
|
||||
|
||||
assert iteration_stats.num_prompt_tokens == total_prompt_tokens
|
||||
assert iteration_stats.num_generation_tokens == num_active
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
|
||||
BaseTokenizerGroup)
|
||||
from vllm.v1.engine import EngineCoreOutput, FinishReason
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
||||
|
||||
GeneralTokenizerType = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
|
||||
|
||||
# Number of sample logprobs to request when testing sample logprobs
|
||||
NUM_SAMPLE_LOGPROBS_UNDER_TEST = 5
|
||||
# Number of prompt logprobs to request when testing prompt logprobs
|
||||
NUM_PROMPT_LOGPROBS_UNDER_TEST = 7
|
||||
|
||||
TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
|
||||
|
||||
FULL_STRINGS = [
|
||||
"My name is Robert from Neural Magic and I love working on vLLM so much!",
|
||||
"Red Hat is the best open source company by far across Linux, K8s, and AI.",
|
||||
"Nick is the name of my brother in addition to my colleague from Red Hat.",
|
||||
]
|
||||
STOP_STRINGS = ["I love working on", "company by far", "brother in"]
|
||||
PROMPT_LEN = 5
|
||||
|
||||
PLP_APC_UNSUPPORTED_MSG = ("Prefix caching with prompt logprobs not yet "
|
||||
"supported on VLLM V1.")
|
||||
|
||||
random.seed(42)
|
||||
|
||||
|
||||
def _create_random_top_logprob_test_vector(
|
||||
num_logprobs: int,
|
||||
lower: float,
|
||||
upper: float,
|
||||
) -> torch.Tensor:
|
||||
"""Create a random vector of top logprob float values.
|
||||
|
||||
Use to create fake sample logprobs for testing.
|
||||
|
||||
Note that a real production scenario would require
|
||||
logprobs to be sorted in descending order, something
|
||||
which is omitted in this function.
|
||||
|
||||
Args:
|
||||
num_logprobs: number of top logprobs
|
||||
lower: lower range of logprob float values
|
||||
upper: upper range of logprob float values
|
||||
|
||||
Returns:
|
||||
1D length-`num_logprobs` torch Tensor of float logprob values
|
||||
"""
|
||||
return torch.rand(num_logprobs) * (upper - lower) + lower
|
||||
|
||||
|
||||
def _create_random_top_logprob_test_matrix(
|
||||
shape: Tuple,
|
||||
lower: float,
|
||||
upper: float,
|
||||
) -> torch.Tensor:
|
||||
"""Create a random matrix of top logprob float values.
|
||||
|
||||
Use to create fake prompt logprobs for testing.
|
||||
|
||||
Note that a real production scenario would require
|
||||
logprobs to be sorted in descending order along rows,
|
||||
something which is omitted in this function.
|
||||
|
||||
Args:
|
||||
shape: (num_tokens,num_logprobs) tuple representing
|
||||
matrix shape
|
||||
lower: lower range of logprob float values
|
||||
upper: upper range of logprob float values
|
||||
|
||||
Returns:
|
||||
2D num_tokens x num_logprobs torch Tensor of float logprob values
|
||||
"""
|
||||
return torch.rand(*shape) * (upper - lower) + lower
|
||||
|
||||
|
||||
def _create_random_top_token_test_vector(
|
||||
num_logprobs: int,
|
||||
lower: int,
|
||||
upper: int,
|
||||
sampled_token_id: int,
|
||||
adjust_num_logprobs: bool = True) -> Tuple[torch.Tensor, int]:
|
||||
"""Create a random vector of top logprob token indices
|
||||
|
||||
Use to create fake sample logprobs for testing. The sampled token
|
||||
ID must always be one of the top logprobs, which this dummy test
|
||||
vector generator enforces. OpenAI API
|
||||
compatible engines must be able to return an additional sample
|
||||
logprob for the sampled token if the sampled token was not
|
||||
among the top sample logprobs; `adjust_num_logprobs` emulates
|
||||
this behavior by increasing the vector length by 1 if
|
||||
`adjust_num_logprobs` is set.
|
||||
|
||||
Args:
|
||||
num_logprobs: number of top logprobs
|
||||
lower: lower range of token ids
|
||||
upper: upper range of token ids
|
||||
sampled_token_id: the token actually sampled
|
||||
adjust_num_logprobs: if True, emulate situation where sampled
|
||||
token logprob must be injected into top
|
||||
logprobs
|
||||
|
||||
Returns:
|
||||
1D length-x torch Tensor of token ids where x is
|
||||
`num_logprobs+1` if `adjust_num_logprobs` and
|
||||
`num_logprobs` otherwise
|
||||
sampled_token_rank: the rank of sampled_token_id in the vocab
|
||||
vector when sorted in descending order by
|
||||
logprob
|
||||
"""
|
||||
|
||||
# Calculate the final number of logprobs required
|
||||
total_logprobs = num_logprobs + 1 if adjust_num_logprobs else num_logprobs
|
||||
|
||||
# Generate random indices using torch
|
||||
choice_tensor = torch.randperm(upper - lower)[:total_logprobs] + lower
|
||||
|
||||
# Ensure the sampled token ID is included in the tensor
|
||||
choice_tensor[0] = sampled_token_id
|
||||
|
||||
# Check if the sampled_token_id occurs in choice_tensor[1:]
|
||||
if sampled_token_id in choice_tensor[1:]:
|
||||
sampled_token_rank = (choice_tensor[1:] == sampled_token_id).nonzero(
|
||||
as_tuple=True)[0].item()
|
||||
else:
|
||||
# If not found, assign a random int between num_logprobs and 50700
|
||||
sampled_token_rank = random.randint(num_logprobs, 50700)
|
||||
|
||||
return choice_tensor, sampled_token_rank
|
||||
|
||||
|
||||
def _create_random_top_token_test_matrix(
|
||||
shape: Tuple[int, int],
|
||||
lower: int,
|
||||
upper: int,
|
||||
tokens_list: List[int],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Create a random matrix of top logprob token indices
|
||||
|
||||
Use to create fake prompt logprobs for testing.
|
||||
|
||||
Token ids are generated randomly and sampled without
|
||||
replacement.
|
||||
|
||||
Args:
|
||||
shape: (num_tokens, num_logprobs) tuple representing
|
||||
matrix shape
|
||||
lower: lower range of token ids
|
||||
upper: upper range of token ids
|
||||
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- 2D num_tokens x num_logprobs+1 torch Tensor of token ids
|
||||
- 1D tensor of ranks of prompt tokens in their respective
|
||||
rows, or random values
|
||||
"""
|
||||
num_elements = shape[0] * shape[1]
|
||||
choice_tensor = torch.randperm(upper - lower)[:num_elements] + lower
|
||||
matrix = torch.cat(
|
||||
(torch.tensor(tokens_list, dtype=torch.int).unsqueeze(-1),
|
||||
choice_tensor.view(shape)),
|
||||
dim=1)
|
||||
|
||||
# Initialize the tensor for storing the ranks
|
||||
prompt_token_ranks = torch.empty(shape[0], dtype=torch.int)
|
||||
|
||||
# Iterate over each row to check presence of
|
||||
# tokens_list[rdx] and determine its index
|
||||
for rdx in range(shape[0]):
|
||||
row = matrix[rdx,
|
||||
1:] # Skip the first column as it contains the token list
|
||||
token_index = (row == tokens_list[rdx]).nonzero(as_tuple=True)[0]
|
||||
if token_index.numel() > 0:
|
||||
prompt_token_ranks[rdx] = token_index.item()
|
||||
else:
|
||||
prompt_token_ranks[rdx] = random.randint(shape[1], 50700)
|
||||
|
||||
return matrix, prompt_token_ranks
|
||||
|
||||
|
||||
def decode_token(
|
||||
tok_id: int,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
) -> str:
|
||||
"""Reproduce the process of detokenizing a token for testing purposes.
|
||||
|
||||
Args:
|
||||
tok_id: token id to detokenize
|
||||
tokenizer: tokenizer to use for detokenization
|
||||
|
||||
Returns:
|
||||
string representation of token
|
||||
"""
|
||||
return tokenizer.convert_ids_to_tokens(tok_id)
|
||||
|
||||
|
||||
def generate_dummy_sample_logprobs(
|
||||
sampled_tokens_list: List,
|
||||
num_logprobs: int,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
) -> List[Tuple[List[int], List[float], int]]:
|
||||
"""Generate dummy sample logprobs
|
||||
|
||||
Generate a test data structure which imitates the list of sample logprobs
|
||||
which would be assembled in the engine core during decode phase.
|
||||
|
||||
Args:
|
||||
sampled_tokens_list: list of sampled tokens
|
||||
num_logprobs: return `num_logprobs` or `num_logprobs+1` logprobs per token
|
||||
tokenizer: model tokenizer to use for detokenization
|
||||
|
||||
Returns
|
||||
List of (top token ids vector, logprobs vector, sampled token rank)
|
||||
Python lists tuples; in each tuple the logprobs and top token ids
|
||||
vectors have the same length which is either `num_logprobs` or
|
||||
`num_logprobs+1`. Sampled token rank is the rank (index+1) of the
|
||||
sampled token within the vocab vector when sorted by logprob in
|
||||
descending order.
|
||||
"""
|
||||
res = []
|
||||
for sampled_token_id in sampled_tokens_list:
|
||||
(
|
||||
token_vector,
|
||||
sampled_token_rank,
|
||||
) = _create_random_top_token_test_vector(num_logprobs, 0,
|
||||
len(tokenizer.vocab) - 1,
|
||||
sampled_token_id)
|
||||
|
||||
res.append(
|
||||
(token_vector,
|
||||
_create_random_top_logprob_test_vector(num_logprobs + 1, -100,
|
||||
0), sampled_token_rank))
|
||||
|
||||
# Convert tensors in the list tuples to Python lists
|
||||
res_list_format = [
|
||||
(log_probs_tensor.tolist(), token_ids_tensor.tolist(),
|
||||
sampled_token_rank)
|
||||
for log_probs_tensor, token_ids_tensor, sampled_token_rank in res
|
||||
]
|
||||
|
||||
return res_list_format
|
||||
|
||||
|
||||
def generate_dummy_prompt_logprobs_tensors(
|
||||
prompt_tokens_list: List,
|
||||
num_logprobs: int,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
) -> LogprobsTensors:
|
||||
"""Generate dummy prompt logprobs tensors
|
||||
|
||||
Generate a test data structure which imitates the torch Tensors of prompt
|
||||
logprobs which would be assembled in the engine core during chunked
|
||||
prefill.
|
||||
|
||||
Args:
|
||||
prompt_tokens_list: list of prompt tokens
|
||||
num_logprobs: return `num_logprobs` logprobs per token
|
||||
tokenizer: model tokenizer to use for detokenization
|
||||
|
||||
Returns
|
||||
Single Tuple of (logprobs matrix, top token ids matrix) torch Tensor,
|
||||
where both matrices have dimensions
|
||||
num_prompt_tokens x num_logprobs
|
||||
"""
|
||||
# For now, assume the whole prompt is processed in one chunk; thus,
|
||||
# the number of non-`None` prompt logprobs is `len(prompt_tokens_list)-1`.
|
||||
# Prior to injecting `None` at the beginning of prompt logprobs (which
|
||||
# happens later in the detokenizer, not here), the prompt logprobs in
|
||||
# the ith position are predicting the probability distribution of the
|
||||
# prompt token in (i+1)st position. Thus, we concat
|
||||
# `prompt_tokens_list[1:]` to the dummy token ids, just as the engine
|
||||
# would.
|
||||
num_prompt_logprobs = len(prompt_tokens_list) - 1
|
||||
(
|
||||
token_vector,
|
||||
prompt_token_ranks,
|
||||
) = _create_random_top_token_test_matrix(
|
||||
(num_prompt_logprobs, num_logprobs), 0,
|
||||
len(tokenizer.vocab) - 1, prompt_tokens_list[1:])
|
||||
return LogprobsTensors(
|
||||
token_vector,
|
||||
_create_random_top_logprob_test_matrix(
|
||||
(num_prompt_logprobs, num_logprobs + 1), -100, 0),
|
||||
prompt_token_ranks)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyOutputProcessorTestVectors:
|
||||
"""Dummy test vectors for output processor tests"""
|
||||
tokenizer: GeneralTokenizerType
|
||||
tokenizer_group: BaseTokenizerGroup
|
||||
vllm_config: EngineArgs
|
||||
full_tokens: List[List[int]] # Prompt + generated tokens
|
||||
prompt_tokens: List[List[int]]
|
||||
generation_tokens: List[List[int]]
|
||||
# Each request is associated with a tuple of
|
||||
# (top tokens, top logprobs, ranks) prompt logprobs tensors
|
||||
prompt_logprobs: List[LogprobsTensors]
|
||||
# Each request is associated with a sample logprobs; a request's
|
||||
# sample logprobs are a list of (top tokens, top logprobs, ranks)
|
||||
# sample logprobs tensors at each sequence position
|
||||
generation_logprobs: List[List[Tuple[List[int], List[float], int]]]
|
||||
prompt_strings: List[str]
|
||||
prompt_strings_len: List[int]
|
||||
generation_strings: List[str]
|
||||
|
||||
|
||||
class MockEngineCore:
|
||||
"""Mock engine core outputs form premade tokens lists."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokens_list: List[List[int]],
|
||||
# For each request, for each sampled token offset,
|
||||
# a tuple of
|
||||
# (list of topk token ids, list of sample logprob vals, rank)
|
||||
generated_logprobs_raw: Optional[List[List[Tuple[List[int],
|
||||
List[float],
|
||||
int]]]] = None,
|
||||
# For each request, a tuple of
|
||||
# (prompt logprob val matrix, prompt logprob tok id matrix);
|
||||
# each matrix has dimensions
|
||||
# (num prompt toks) x (num prompt logprobs+1)
|
||||
prompt_logprobs_raw: Optional[List[LogprobsTensors]] = None,
|
||||
) -> None:
|
||||
self.tokens_list = tokens_list
|
||||
self.current_idx = 0
|
||||
self.generated_logprobs_raw = generated_logprobs_raw
|
||||
self.do_logprobs = generated_logprobs_raw is not None
|
||||
self.prompt_logprobs_raw = prompt_logprobs_raw
|
||||
self.do_prompt_logprobs = prompt_logprobs_raw is not None
|
||||
|
||||
def get_outputs(self) -> List[EngineCoreOutput]:
|
||||
do_logprobs = self.do_logprobs
|
||||
do_prompt_logprobs = self.do_prompt_logprobs
|
||||
token_idx = self.current_idx
|
||||
|
||||
outputs = []
|
||||
for req_idx, token_ids in enumerate(self.tokens_list):
|
||||
if len(token_ids) > token_idx:
|
||||
if do_logprobs:
|
||||
assert self.generated_logprobs_raw is not None
|
||||
(logprobs_token_ids_, logprobs_, sampled_token_ranks_) = (
|
||||
self.generated_logprobs_raw[req_idx][token_idx])
|
||||
logprobs = LogprobsLists(
|
||||
[logprobs_token_ids_],
|
||||
[logprobs_],
|
||||
[sampled_token_ranks_],
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
if do_prompt_logprobs:
|
||||
if self.current_idx == 0:
|
||||
assert self.prompt_logprobs_raw is not None
|
||||
prompt_logprobs = self.prompt_logprobs_raw[req_idx]
|
||||
else:
|
||||
prompt_logprobs = None
|
||||
else:
|
||||
prompt_logprobs = None
|
||||
output = EngineCoreOutput(
|
||||
request_id=f"request-{req_idx}",
|
||||
new_token_ids=[token_ids[token_idx]],
|
||||
new_logprobs=logprobs,
|
||||
new_prompt_logprobs_tensors=prompt_logprobs,
|
||||
)
|
||||
if token_idx == len(token_ids) - 1:
|
||||
output.finish_reason = FinishReason.STOP
|
||||
outputs.append(output)
|
||||
|
||||
self.current_idx += 1
|
||||
return outputs
|
||||
Reference in New Issue
Block a user