Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+308
-267
File diff suppressed because it is too large
Load Diff
@@ -27,10 +27,9 @@ class MedusaProposer:
|
||||
# Save config parameters
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.max_num_tokens = (
|
||||
vllm_config.scheduler_config.max_num_batched_tokens)
|
||||
self.hidden_size = vllm_config.speculative_config.\
|
||||
draft_model_config.get_hidden_size(
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.hidden_size = (
|
||||
vllm_config.speculative_config.draft_model_config.get_hidden_size()
|
||||
)
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
|
||||
@@ -51,16 +50,19 @@ class MedusaProposer:
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
with set_model_tag("medusa_head"):
|
||||
self.model = get_model(vllm_config=self.vllm_config,
|
||||
model_config=self.vllm_config.
|
||||
speculative_config.draft_model_config)
|
||||
self.model = get_model(
|
||||
vllm_config=self.vllm_config,
|
||||
model_config=self.vllm_config.speculative_config.draft_model_config,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def dummy_run(self, num_tokens: int) -> None:
|
||||
hidden_states = torch.zeros((self.max_num_tokens, self.hidden_size),
|
||||
dtype=self.dtype,
|
||||
device=self.device)
|
||||
with set_forward_context(None, self.vllm_config,
|
||||
num_tokens=num_tokens):
|
||||
hidden_states = torch.zeros(
|
||||
(self.max_num_tokens, self.hidden_size),
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
with set_forward_context(None, self.vllm_config, num_tokens=num_tokens):
|
||||
self.model(hidden_states)
|
||||
|
||||
@@ -8,7 +8,6 @@ import torch
|
||||
|
||||
@dataclass
|
||||
class SpecDecodeMetadata:
|
||||
|
||||
# [num_tokens]
|
||||
draft_token_ids: torch.Tensor
|
||||
# [batch_size]
|
||||
@@ -36,22 +35,19 @@ class SpecDecodeMetadata:
|
||||
flattened_draft_token_ids = sum(draft_token_ids, [])
|
||||
num_tokens = len(flattened_draft_token_ids)
|
||||
|
||||
draft_token_ids_tensor = torch.tensor(flattened_draft_token_ids,
|
||||
dtype=torch.int32,
|
||||
device=device)
|
||||
draft_token_ids_tensor = torch.tensor(
|
||||
flattened_draft_token_ids, dtype=torch.int32, device=device
|
||||
)
|
||||
cu_num_draft_tokens = np.cumsum(num_draft_tokens, dtype=np.int32)
|
||||
cu_num_draft_tokens_tensor = torch.from_numpy(cu_num_draft_tokens).to(
|
||||
device)
|
||||
cu_num_draft_tokens_tensor = torch.from_numpy(cu_num_draft_tokens).to(device)
|
||||
|
||||
target_logits_indices = torch.zeros(num_tokens,
|
||||
dtype=torch.int32,
|
||||
device=device)
|
||||
bonus_logits_indices = torch.zeros(batch_size,
|
||||
dtype=torch.int32,
|
||||
device=device)
|
||||
logits_indices = torch.zeros(num_tokens + batch_size,
|
||||
dtype=torch.int32,
|
||||
device=device)
|
||||
target_logits_indices = torch.zeros(
|
||||
num_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
bonus_logits_indices = torch.zeros(batch_size, dtype=torch.int32, device=device)
|
||||
logits_indices = torch.zeros(
|
||||
num_tokens + batch_size, dtype=torch.int32, device=device
|
||||
)
|
||||
return cls(
|
||||
draft_token_ids=draft_token_ids_tensor,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
|
||||
@@ -31,8 +31,10 @@ class SpecDecodingStats:
|
||||
|
||||
@classmethod
|
||||
def new(cls, num_spec_tokens: int) -> "SpecDecodingStats":
|
||||
return cls(num_spec_tokens=num_spec_tokens,
|
||||
num_accepted_tokens_per_pos=[0] * num_spec_tokens)
|
||||
return cls(
|
||||
num_spec_tokens=num_spec_tokens,
|
||||
num_accepted_tokens_per_pos=[0] * num_spec_tokens,
|
||||
)
|
||||
|
||||
def observe_draft(self, num_draft_tokens: int, num_accepted_tokens: int):
|
||||
self.num_drafts += 1
|
||||
@@ -64,10 +66,10 @@ class SpecDecodingLogging:
|
||||
def observe(self, spec_decoding_stats: SpecDecodingStats):
|
||||
self.num_drafts.append(spec_decoding_stats.num_drafts)
|
||||
self.num_draft_tokens.append(spec_decoding_stats.num_draft_tokens)
|
||||
self.num_accepted_tokens.append(
|
||||
spec_decoding_stats.num_accepted_tokens)
|
||||
self.num_accepted_tokens.append(spec_decoding_stats.num_accepted_tokens)
|
||||
self.accepted_tokens_per_pos_lists.append(
|
||||
spec_decoding_stats.num_accepted_tokens_per_pos)
|
||||
spec_decoding_stats.num_accepted_tokens_per_pos
|
||||
)
|
||||
|
||||
def log(self, log_fn=logger.info):
|
||||
if not self.num_drafts:
|
||||
@@ -83,8 +85,11 @@ class SpecDecodingLogging:
|
||||
draft_throughput = num_draft_tokens / elapsed_time
|
||||
accepted_throughput = num_accepted_tokens / elapsed_time
|
||||
|
||||
draft_acceptance_rate = (num_accepted_tokens / num_draft_tokens *
|
||||
100 if num_draft_tokens > 0 else float("nan"))
|
||||
draft_acceptance_rate = (
|
||||
num_accepted_tokens / num_draft_tokens * 100
|
||||
if num_draft_tokens > 0
|
||||
else float("nan")
|
||||
)
|
||||
|
||||
# Conventionally, mean acceptance length includes the bonus token
|
||||
mean_acceptance_length = 1 + (num_accepted_tokens / num_drafts)
|
||||
@@ -149,27 +154,36 @@ class SpecDecodingProm:
|
||||
counter_drafts = self._counter_cls(
|
||||
name="vllm:spec_decode_num_drafts",
|
||||
documentation="Number of spec decoding drafts.",
|
||||
labelnames=labelnames)
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_spec_decode_num_drafts = make_per_engine(
|
||||
counter_drafts, per_engine_labelvalues)
|
||||
counter_drafts, per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_draft_tokens = self._counter_cls(
|
||||
name="vllm:spec_decode_num_draft_tokens",
|
||||
documentation="Number of draft tokens.",
|
||||
labelnames=labelnames)
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_spec_decode_num_draft_tokens = make_per_engine(
|
||||
counter_draft_tokens, per_engine_labelvalues)
|
||||
counter_draft_tokens, per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_accepted_tokens = self._counter_cls(
|
||||
name="vllm:spec_decode_num_accepted_tokens",
|
||||
documentation="Number of accepted tokens.",
|
||||
labelnames=labelnames)
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_spec_decode_num_accepted_tokens = make_per_engine(
|
||||
counter_accepted_tokens, per_engine_labelvalues)
|
||||
counter_accepted_tokens, per_engine_labelvalues
|
||||
)
|
||||
|
||||
assert speculative_config is not None
|
||||
num_spec_tokens = (speculative_config.num_speculative_tokens
|
||||
if self.spec_decoding_enabled else 0)
|
||||
num_spec_tokens = (
|
||||
speculative_config.num_speculative_tokens
|
||||
if self.spec_decoding_enabled
|
||||
else 0
|
||||
)
|
||||
pos_labelnames = labelnames + ["position"]
|
||||
base_counter = self._counter_cls(
|
||||
name="vllm:spec_decode_num_accepted_tokens_per_pos",
|
||||
@@ -177,33 +191,33 @@ class SpecDecodingProm:
|
||||
labelnames=pos_labelnames,
|
||||
)
|
||||
self.counter_spec_decode_num_accepted_tokens_per_pos: dict[
|
||||
int, list[prometheus_client.Counter]] = {
|
||||
idx: [
|
||||
base_counter.labels(*lv, str(pos))
|
||||
for pos in range(num_spec_tokens)
|
||||
]
|
||||
for idx, lv in per_engine_labelvalues.items()
|
||||
}
|
||||
int, list[prometheus_client.Counter]
|
||||
] = {
|
||||
idx: [base_counter.labels(*lv, str(pos)) for pos in range(num_spec_tokens)]
|
||||
for idx, lv in per_engine_labelvalues.items()
|
||||
}
|
||||
|
||||
def observe(self,
|
||||
spec_decoding_stats: SpecDecodingStats,
|
||||
engine_idx: int = 0):
|
||||
def observe(self, spec_decoding_stats: SpecDecodingStats, engine_idx: int = 0):
|
||||
if not self.spec_decoding_enabled:
|
||||
return
|
||||
self.counter_spec_decode_num_drafts[engine_idx].inc(
|
||||
spec_decoding_stats.num_drafts)
|
||||
spec_decoding_stats.num_drafts
|
||||
)
|
||||
self.counter_spec_decode_num_draft_tokens[engine_idx].inc(
|
||||
spec_decoding_stats.num_draft_tokens)
|
||||
spec_decoding_stats.num_draft_tokens
|
||||
)
|
||||
self.counter_spec_decode_num_accepted_tokens[engine_idx].inc(
|
||||
spec_decoding_stats.num_accepted_tokens)
|
||||
spec_decoding_stats.num_accepted_tokens
|
||||
)
|
||||
for pos, counter in enumerate(
|
||||
self.
|
||||
counter_spec_decode_num_accepted_tokens_per_pos[engine_idx]):
|
||||
self.counter_spec_decode_num_accepted_tokens_per_pos[engine_idx]
|
||||
):
|
||||
counter.inc(spec_decoding_stats.num_accepted_tokens_per_pos[pos])
|
||||
|
||||
|
||||
def make_per_engine(counter: prometheus_client.Counter,
|
||||
per_engine_labelvalues: dict[int, list[str]]):
|
||||
def make_per_engine(
|
||||
counter: prometheus_client.Counter, per_engine_labelvalues: dict[int, list[str]]
|
||||
):
|
||||
"""Create a counter for each label value."""
|
||||
return {
|
||||
idx: counter.labels(*labelvalues)
|
||||
|
||||
@@ -9,7 +9,6 @@ from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class NgramProposer:
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
assert vllm_config.speculative_config is not None
|
||||
assert vllm_config.speculative_config.prompt_lookup_min is not None
|
||||
@@ -28,8 +27,7 @@ class NgramProposer:
|
||||
|
||||
# Pre-allocate buffers for numba batch propose.
|
||||
max_num_seqs = vllm_config.scheduler_config.max_num_seqs
|
||||
self.valid_ngram_draft = np.zeros((max_num_seqs, self.k),
|
||||
dtype=np.int32)
|
||||
self.valid_ngram_draft = np.zeros((max_num_seqs, self.k), dtype=np.int32)
|
||||
self.valid_ngram_num_drafts = np.zeros((max_num_seqs), dtype=np.int32)
|
||||
|
||||
# Threshold of total number of tokens in the batch to enable
|
||||
@@ -55,9 +53,13 @@ class NgramProposer:
|
||||
|
||||
# Trigger Numba JIT compilation for N-gram proposer.
|
||||
# This usually takes less than 1 second.
|
||||
self.propose([[]] * 1024, [""] * 1024, np.zeros(1024, dtype=np.int32),
|
||||
np.zeros((1024, self.max_model_len), dtype=np.int32),
|
||||
set())
|
||||
self.propose(
|
||||
[[]] * 1024,
|
||||
[""] * 1024,
|
||||
np.zeros(1024, dtype=np.int32),
|
||||
np.zeros((1024, self.max_model_len), dtype=np.int32),
|
||||
set(),
|
||||
)
|
||||
|
||||
def batch_propose(
|
||||
self,
|
||||
@@ -67,20 +69,20 @@ class NgramProposer:
|
||||
token_ids_cpu: np.ndarray,
|
||||
) -> list[list[int]]:
|
||||
"""Batch version of ngram proposer using numba for acceleration.
|
||||
|
||||
|
||||
Args:
|
||||
valid_ngram_requests:
|
||||
valid_ngram_requests:
|
||||
Set of indices of requests that need ngram proposals.
|
||||
num_tokens_no_spec:
|
||||
Numpy array of shape (batch_size,) representing the number
|
||||
num_tokens_no_spec:
|
||||
Numpy array of shape (batch_size,) representing the number
|
||||
of tokens without speculative tokens for each request.
|
||||
token_ids_cpu:
|
||||
Numpy array of shape (batch_size, max_model_len)
|
||||
token_ids_cpu:
|
||||
Numpy array of shape (batch_size, max_model_len)
|
||||
representing the token IDs for each request.
|
||||
|
||||
Returns:
|
||||
list[list[int]]:
|
||||
A list where each element is a list of proposed
|
||||
list[list[int]]:
|
||||
A list where each element is a list of proposed
|
||||
token IDs for the corresponding request.
|
||||
"""
|
||||
draft_token_ids: list[list[int]] = []
|
||||
@@ -96,26 +98,32 @@ class NgramProposer:
|
||||
total_tokens = np.sum(num_tokens_no_spec)
|
||||
if total_tokens >= self.num_tokens_threshold:
|
||||
final_num_threads = max(
|
||||
1, min(self.num_numba_thread_available,
|
||||
num_ngram_requests))
|
||||
1, min(self.num_numba_thread_available, num_ngram_requests)
|
||||
)
|
||||
set_num_threads(final_num_threads)
|
||||
else:
|
||||
set_num_threads(1)
|
||||
|
||||
batch_propose_numba(valid_ngram_requests, num_tokens_no_spec,
|
||||
token_ids_cpu, self.min_n, self.max_n,
|
||||
self.max_model_len, self.k,
|
||||
self.valid_ngram_draft,
|
||||
self.valid_ngram_num_drafts)
|
||||
batch_propose_numba(
|
||||
valid_ngram_requests,
|
||||
num_tokens_no_spec,
|
||||
token_ids_cpu,
|
||||
self.min_n,
|
||||
self.max_n,
|
||||
self.max_model_len,
|
||||
self.k,
|
||||
self.valid_ngram_draft,
|
||||
self.valid_ngram_num_drafts,
|
||||
)
|
||||
|
||||
# Restore original number of threads.
|
||||
set_num_threads(original_num_numba_threads)
|
||||
|
||||
for i in range(num_requests):
|
||||
if i in valid_ngram_requests and \
|
||||
self.valid_ngram_num_drafts[i] > 0:
|
||||
draft_token_ids.append(self.valid_ngram_draft[
|
||||
i, :self.valid_ngram_num_drafts[i]].tolist())
|
||||
if i in valid_ngram_requests and self.valid_ngram_num_drafts[i] > 0:
|
||||
draft_token_ids.append(
|
||||
self.valid_ngram_draft[i, : self.valid_ngram_num_drafts[i]].tolist()
|
||||
)
|
||||
else:
|
||||
draft_token_ids.append([])
|
||||
|
||||
@@ -129,7 +137,6 @@ class NgramProposer:
|
||||
token_ids_cpu: np.ndarray,
|
||||
spec_decode_unsupported_reqs: set,
|
||||
) -> list[list[int]]:
|
||||
|
||||
# find which requests need ngram proposals
|
||||
valid_ngram_requests = []
|
||||
for i, sampled_ids in enumerate(sampled_token_ids):
|
||||
@@ -166,12 +173,17 @@ class NgramProposer:
|
||||
|
||||
|
||||
@njit(parallel=True)
|
||||
def batch_propose_numba(valid_ngram_requests: list,
|
||||
num_tokens_no_spec: np.ndarray,
|
||||
token_ids_cpu: np.ndarray, min_n: int, max_n: int,
|
||||
max_model_len: int, k: int,
|
||||
valid_ngram_draft: np.ndarray,
|
||||
valid_ngram_num_drafts: np.ndarray):
|
||||
def batch_propose_numba(
|
||||
valid_ngram_requests: list,
|
||||
num_tokens_no_spec: np.ndarray,
|
||||
token_ids_cpu: np.ndarray,
|
||||
min_n: int,
|
||||
max_n: int,
|
||||
max_model_len: int,
|
||||
k: int,
|
||||
valid_ngram_draft: np.ndarray,
|
||||
valid_ngram_num_drafts: np.ndarray,
|
||||
):
|
||||
for i in prange(len(valid_ngram_requests)):
|
||||
idx = valid_ngram_requests[i]
|
||||
num_tokens = num_tokens_no_spec[idx]
|
||||
@@ -181,19 +193,22 @@ def batch_propose_numba(valid_ngram_requests: list,
|
||||
min_ngram=min_n,
|
||||
max_ngram=max_n,
|
||||
max_model_len=max_model_len,
|
||||
k=k)
|
||||
k=k,
|
||||
)
|
||||
|
||||
valid_ngram_num_drafts[i] = drafter_output.shape[0]
|
||||
if len(drafter_output):
|
||||
valid_ngram_draft[i, :drafter_output.shape[0]] = drafter_output
|
||||
valid_ngram_draft[i, : drafter_output.shape[0]] = drafter_output
|
||||
|
||||
|
||||
@jit(nopython=True)
|
||||
def _find_longest_matched_ngram_and_propose_tokens(origin_tokens: np.ndarray,
|
||||
min_ngram: int,
|
||||
max_ngram: int,
|
||||
max_model_len: int,
|
||||
k: int) -> np.ndarray:
|
||||
def _find_longest_matched_ngram_and_propose_tokens(
|
||||
origin_tokens: np.ndarray,
|
||||
min_ngram: int,
|
||||
max_ngram: int,
|
||||
max_model_len: int,
|
||||
k: int,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Find the longest n-gram which matches the suffix of the given tokens
|
||||
whose length is within [min_ngram, max_ngram] (inclusive).
|
||||
@@ -203,12 +218,12 @@ def _find_longest_matched_ngram_and_propose_tokens(origin_tokens: np.ndarray,
|
||||
# Do not generate draft tokens is context is shorter than minimum n-gram
|
||||
total_token = origin_tokens.shape[0]
|
||||
if total_token < min_ngram:
|
||||
return np.empty((0, ), dtype=origin_tokens.dtype)
|
||||
return np.empty((0,), dtype=origin_tokens.dtype)
|
||||
|
||||
# Do not generate draft tokens beyond the max model length.
|
||||
k = min(k, max_model_len - total_token)
|
||||
if k <= 0:
|
||||
return np.empty((0, ), dtype=origin_tokens.dtype)
|
||||
return np.empty((0,), dtype=origin_tokens.dtype)
|
||||
|
||||
# Flip tokens, and the goal become to find longest ngram
|
||||
# on the rightmost position which matches the prefix with
|
||||
@@ -265,7 +280,7 @@ def _find_longest_matched_ngram_and_propose_tokens(origin_tokens: np.ndarray,
|
||||
|
||||
if longest_ngram < min_ngram:
|
||||
# No valid ngram is found
|
||||
return np.empty((0, ), dtype=origin_tokens.dtype)
|
||||
return np.empty((0,), dtype=origin_tokens.dtype)
|
||||
|
||||
# Flip the position back, so in origin_tokens,
|
||||
# origin_tokens[total_token-1-position:total_token-1-position+longest_ngram]
|
||||
@@ -273,4 +288,4 @@ def _find_longest_matched_ngram_and_propose_tokens(origin_tokens: np.ndarray,
|
||||
# total_token-1-position+longest_ngram
|
||||
start_position = total_token - 1 - position + longest_ngram
|
||||
k = min(k, total_token - start_position)
|
||||
return origin_tokens[start_position:start_position + k]
|
||||
return origin_tokens[start_position : start_position + k]
|
||||
|
||||
@@ -7,8 +7,10 @@ _SAMPLING_EPS = 1e-5
|
||||
|
||||
def is_spec_decode_unsupported(sampling_params: SamplingParams) -> bool:
|
||||
"""True if request is incompatible with speculative decoding"""
|
||||
return (sampling_params.frequency_penalty != 0.0
|
||||
or sampling_params.presence_penalty != 0.0
|
||||
or sampling_params.repetition_penalty != 1.0
|
||||
or sampling_params.min_p > _SAMPLING_EPS
|
||||
or sampling_params.logprobs is not None)
|
||||
return (
|
||||
sampling_params.frequency_penalty != 0.0
|
||||
or sampling_params.presence_penalty != 0.0
|
||||
or sampling_params.repetition_penalty != 1.0
|
||||
or sampling_params.min_p > _SAMPLING_EPS
|
||||
or sampling_params.logprobs is not None
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user