Merge branch 'main' into patches
This commit is contained in:
@@ -1036,26 +1036,22 @@ def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) ->
|
||||
|
||||
def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
|
||||
"""Allocate the smallest KV cache that still lets every graph be captured."""
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
get_kv_cache_config_from_groups,
|
||||
get_kv_cache_groups,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
|
||||
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups_from_builder
|
||||
|
||||
kv_cache_spec = runner.get_kv_cache_spec()
|
||||
kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec)
|
||||
builder = get_kv_cache_config_builder(runner.vllm_config)
|
||||
kv_cache_groups = get_kv_cache_groups_from_builder(
|
||||
builder, runner.vllm_config, kv_cache_spec
|
||||
)
|
||||
# At least one block per sequence is required to capture the graphs.
|
||||
min_blocks = (
|
||||
min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size)
|
||||
or 1
|
||||
)
|
||||
saved_override = runner.cache_config.num_gpu_blocks_override
|
||||
runner.cache_config.num_gpu_blocks_override = min_blocks
|
||||
try:
|
||||
minimal_config = get_kv_cache_config_from_groups(
|
||||
runner.vllm_config, kv_cache_groups, available_memory=0
|
||||
)
|
||||
finally:
|
||||
runner.cache_config.num_gpu_blocks_override = saved_override
|
||||
minimal_config = builder.get_kv_cache_config_from_groups(
|
||||
runner.vllm_config, kv_cache_groups, num_blocks=min_blocks
|
||||
)
|
||||
|
||||
runner.initialize_kv_cache(minimal_config, is_profiling=True)
|
||||
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
|
||||
|
||||
@@ -298,8 +298,8 @@ class Sampler:
|
||||
seq_lens_upper_bound_np,
|
||||
skip_top_k_top_p=True,
|
||||
)
|
||||
top_k, top_p = self.sampling_states.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np
|
||||
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np, logits.shape[-1]
|
||||
)
|
||||
# Don't use a fused sampler if no requests use top_k/top_p, if there are
|
||||
# any greedy requests or per-request seeds, or if post-processed
|
||||
@@ -321,6 +321,7 @@ class Sampler:
|
||||
pos,
|
||||
top_k,
|
||||
top_p,
|
||||
has_topp_only_rows,
|
||||
use_fused_sampler,
|
||||
)
|
||||
|
||||
@@ -332,6 +333,7 @@ class Sampler:
|
||||
pos: torch.Tensor,
|
||||
top_k: torch.Tensor | None,
|
||||
top_p: torch.Tensor | None,
|
||||
has_topp_only_rows: bool,
|
||||
use_fused_sampler: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if use_fused_sampler:
|
||||
@@ -342,7 +344,9 @@ class Sampler:
|
||||
else: # Use XPU sampler
|
||||
sampled, _ = xpu_sample(processed_logits, top_k, top_p)
|
||||
else:
|
||||
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
|
||||
processed_logits = apply_top_k_top_p(
|
||||
processed_logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
sampled = gumbel_sample(
|
||||
processed_logits,
|
||||
expanded_idx_mapping,
|
||||
|
||||
@@ -103,13 +103,22 @@ class SamplingStates:
|
||||
apply_min_p(logits, expanded_idx_mapping, self.min_p.gpu)
|
||||
|
||||
def get_top_k_top_p(
|
||||
self, expanded_idx_mapping: torch.Tensor, idx_mapping_np: np.ndarray
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size)
|
||||
do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0)
|
||||
self,
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
idx_mapping_np: np.ndarray,
|
||||
vocab_size: int,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None, bool]:
|
||||
"""Returns (top_k, top_p, has_topp_only_rows)."""
|
||||
top_k_np = self.top_k.np[idx_mapping_np]
|
||||
top_p_np = self.top_p.np[idx_mapping_np]
|
||||
do_top_k = np.any(top_k_np != self.vocab_size)
|
||||
do_top_p = np.any(top_p_np != 1.0)
|
||||
top_k = self.top_k.gpu[expanded_idx_mapping] if do_top_k else None
|
||||
top_p = self.top_p.gpu[expanded_idx_mapping] if do_top_p else None
|
||||
return top_k, top_p
|
||||
has_topp_only_rows = bool(
|
||||
do_top_p and np.any((top_k_np >= vocab_size) & (top_p_np < 1.0))
|
||||
)
|
||||
return top_k, top_p, has_topp_only_rows
|
||||
|
||||
def apply_top_k_top_p(
|
||||
self,
|
||||
@@ -117,10 +126,14 @@ class SamplingStates:
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
idx_mapping_np: np.ndarray,
|
||||
) -> torch.Tensor:
|
||||
top_k, top_p = self.get_top_k_top_p(expanded_idx_mapping, idx_mapping_np)
|
||||
top_k, top_p, has_topp_only_rows = self.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np, logits.shape[-1]
|
||||
)
|
||||
if top_k is None and top_p is None:
|
||||
return logits
|
||||
return apply_top_k_top_p(logits, top_k, top_p)
|
||||
return apply_top_k_top_p(
|
||||
logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
|
||||
def any_greedy(self, idx_mapping_np: np.ndarray) -> bool:
|
||||
return bool(np.any(self.temperature.np[idx_mapping_np] == 0.0))
|
||||
|
||||
@@ -194,6 +194,16 @@ def _compute_global_logprobs_and_logsumexp(
|
||||
return target_log_prob, draft_log_prob, target_lse, draft_lse
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _is_bonus_logit(expanded_local_pos_ptr, logit_idx, num_logits):
|
||||
next_local_pos = tl.load(
|
||||
expanded_local_pos_ptr + logit_idx + 1,
|
||||
mask=logit_idx + 1 < num_logits,
|
||||
other=0,
|
||||
)
|
||||
return next_local_pos == 0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_local_logits_stats_kernel(
|
||||
# [num_logits, num_blocks]
|
||||
@@ -225,17 +235,17 @@ def _compute_local_logits_stats_kernel(
|
||||
# [max_num_reqs]
|
||||
temp_ptr,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_DRAFT_LOGITS: tl.constexpr,
|
||||
):
|
||||
logit_idx = tl.program_id(0).to(tl.int64)
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
|
||||
if draft_step_idx >= num_speculative_steps:
|
||||
# Bonus token. Max/argmax and summed exponentials are not needed.
|
||||
if _is_bonus_logit(expanded_local_pos_ptr, logit_idx, num_logits):
|
||||
# Max/argmax and summed exponentials are not needed.
|
||||
return
|
||||
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + logit_idx).to(tl.int64)
|
||||
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
|
||||
|
||||
@@ -423,17 +433,17 @@ def _compute_local_residual_mass_kernel(
|
||||
# [max_num_reqs]
|
||||
temp_ptr,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
vocab_num_blocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
|
||||
):
|
||||
logit_idx = tl.program_id(0).to(tl.int64)
|
||||
# Adaptive verification can end before num_speculative_steps.
|
||||
if logit_idx + 1 >= tl.num_programs(0):
|
||||
return
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
if draft_step_idx == 0 or draft_step_idx >= num_speculative_steps:
|
||||
if draft_step_idx == 0 or _is_bonus_logit(
|
||||
expanded_local_pos_ptr, logit_idx, num_logits
|
||||
):
|
||||
# The acceptance threshold, h, looks one position ahead and sums
|
||||
# over: max(p_i * M_b(x|x_{<i}) - M_s(x|x_{<i}), 0). Tokens at the
|
||||
# first and last (bonus) positions aren't needed for this computation.
|
||||
@@ -1138,7 +1148,7 @@ def rejection_sample(
|
||||
expanded_local_pos,
|
||||
temperature,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
|
||||
HAS_DRAFT_LOGITS=has_draft_logits,
|
||||
)
|
||||
@@ -1209,7 +1219,7 @@ def rejection_sample(
|
||||
expanded_local_pos,
|
||||
temperature,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
vocab_num_blocks,
|
||||
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
|
||||
PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks,
|
||||
|
||||
Reference in New Issue
Block a user