[Model Runner V2] Use FlashInfer sampler (#42472)
This commit is contained in:
@@ -19,6 +19,69 @@ if HAS_TRITON:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_FLASHINFER_MIN_VERSION = "0.2.3"
|
||||
|
||||
|
||||
def flashinfer_sampler_supported() -> bool:
|
||||
"""Decide whether FlashInfer's top-p/top-k sampler can be used.
|
||||
|
||||
Returns False (with appropriate logging) when ``VLLM_USE_FLASHINFER_SAMPLER``
|
||||
is 0, when the platform isn't CUDA, when the GPU's compute capability is
|
||||
unsupported, or when the installed flashinfer is missing or too old. Raises
|
||||
``RuntimeError`` if the user explicitly opted in via the env var but
|
||||
FlashInfer is unavailable.
|
||||
|
||||
Note: callers must additionally ensure ``logprobs_mode`` doesn't require
|
||||
post-top-k/top-p logits/logprobs for any request whose logprobs will be
|
||||
returned in this step, since FlashInfer doesn't expose those.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
return False
|
||||
if not envs.VLLM_USE_FLASHINFER_SAMPLER:
|
||||
logger.info_once(
|
||||
"FlashInfer top-p/top-k sampling disabled via "
|
||||
"VLLM_USE_FLASHINFER_SAMPLER=0."
|
||||
)
|
||||
return False
|
||||
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
assert capability is not None
|
||||
unsupported_reason: str | None = None
|
||||
if not FlashInferBackend.supports_compute_capability(capability):
|
||||
unsupported_reason = (
|
||||
f"unsupported compute capability {capability.as_version_str()}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
import flashinfer
|
||||
|
||||
if version.parse(flashinfer.__version__) < version.parse(
|
||||
_FLASHINFER_MIN_VERSION
|
||||
):
|
||||
unsupported_reason = (
|
||||
f"flashinfer {flashinfer.__version__} is too old "
|
||||
f"(>={_FLASHINFER_MIN_VERSION} required)"
|
||||
)
|
||||
except ImportError:
|
||||
unsupported_reason = "flashinfer is not installed"
|
||||
|
||||
if unsupported_reason is None:
|
||||
logger.info_once("Using FlashInfer for top-p & top-k sampling.", scope="global")
|
||||
return True
|
||||
if envs.is_set("VLLM_USE_FLASHINFER_SAMPLER"):
|
||||
raise RuntimeError(
|
||||
f"FlashInfer top-p/top-k sampling unavailable: {unsupported_reason}. "
|
||||
"Unset VLLM_USE_FLASHINFER_SAMPLER=1."
|
||||
)
|
||||
logger.warning_once(
|
||||
"FlashInfer top-p/top-k sampling unavailable: %s; falling back. "
|
||||
"Set VLLM_USE_FLASHINFER_SAMPLER=0 to silence.",
|
||||
unsupported_reason,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
class TopKTopPSampler(nn.Module):
|
||||
"""
|
||||
Module that performs optional top-k and top-p filtering followed by
|
||||
@@ -30,49 +93,16 @@ class TopKTopPSampler(nn.Module):
|
||||
def __init__(self, logprobs_mode: LogprobsMode = "raw_logprobs") -> None:
|
||||
super().__init__()
|
||||
self.logprobs_mode = logprobs_mode
|
||||
# flashinfer optimization does not apply if intermediate
|
||||
# logprobs/logits after top_k/top_p need to be returned
|
||||
if (
|
||||
logprobs_mode not in ("processed_logits", "processed_logprobs")
|
||||
and current_platform.is_cuda()
|
||||
):
|
||||
if envs.VLLM_USE_FLASHINFER_SAMPLER:
|
||||
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
assert capability is not None
|
||||
if FlashInferBackend.supports_compute_capability(capability):
|
||||
logger.info_once(
|
||||
"Using FlashInfer for top-p & top-k sampling.",
|
||||
scope="global",
|
||||
)
|
||||
self.forward = self.forward_cuda
|
||||
elif envs.is_set("VLLM_USE_FLASHINFER_SAMPLER"):
|
||||
# User explicitly opted in but the GPU can't run FlashInfer.
|
||||
capability_str = capability.as_version_str()
|
||||
raise RuntimeError(
|
||||
"FlashInfer does not support compute capability "
|
||||
f"{capability_str}, unset VLLM_USE_FLASHINFER_SAMPLER=1."
|
||||
)
|
||||
else:
|
||||
# Default-on path; hardware can't run FlashInfer →
|
||||
# quietly fall back to the PyTorch-native sampler
|
||||
# instead of failing server startup.
|
||||
logger.warning_once(
|
||||
"FlashInfer top-p/top-k sampling not supported on "
|
||||
"compute capability %s; falling back to PyTorch-native "
|
||||
"sampler. Set VLLM_USE_FLASHINFER_SAMPLER=0 to silence.",
|
||||
capability.as_version_str(),
|
||||
)
|
||||
self.forward = self.forward_native
|
||||
else:
|
||||
# User explicitly set VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
logger.info_once(
|
||||
"FlashInfer top-p/top-k sampling disabled via "
|
||||
"VLLM_USE_FLASHINFER_SAMPLER=0; using PyTorch-native sampler."
|
||||
)
|
||||
self.forward = self.forward_native
|
||||
|
||||
if current_platform.is_cuda():
|
||||
# FlashInfer doesn't expose post-top-k/top-p logits/logprobs,
|
||||
# so it can't be used when the configured mode requires them.
|
||||
can_use_flashinfer = (
|
||||
logprobs_mode not in ("processed_logits", "processed_logprobs")
|
||||
and flashinfer_sampler_supported()
|
||||
)
|
||||
self.forward = (
|
||||
self.forward_cuda if can_use_flashinfer else self.forward_native
|
||||
)
|
||||
elif current_platform.is_cpu():
|
||||
arch = current_platform.get_cpu_architecture()
|
||||
# Fall back to native implementation for POWERPC and RISCV.
|
||||
@@ -417,7 +447,7 @@ def flashinfer_sample(
|
||||
logits: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
generators: dict[int, torch.Generator],
|
||||
generators: dict[int, torch.Generator] = {}, # noqa
|
||||
) -> torch.Tensor:
|
||||
"""Sample from the logits using FlashInfer.
|
||||
|
||||
@@ -431,11 +461,6 @@ def flashinfer_sample(
|
||||
"""
|
||||
import flashinfer
|
||||
|
||||
if version.parse(flashinfer.__version__) < version.parse("0.2.3"):
|
||||
raise ImportError(
|
||||
"FlashInfer version >= 0.2.3 required for top-k and top-p sampling. "
|
||||
)
|
||||
|
||||
assert not (k is None and p is None)
|
||||
if k is None:
|
||||
# Top-p only.
|
||||
|
||||
@@ -7,6 +7,11 @@ import torch
|
||||
import vllm.envs as envs
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import (
|
||||
apply_top_k_top_p,
|
||||
flashinfer_sample,
|
||||
flashinfer_sampler_supported,
|
||||
)
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.metrics.logits import get_num_nans
|
||||
from vllm.v1.worker.gpu.sample.bad_words import BadWordsState
|
||||
@@ -45,6 +50,7 @@ class Sampler:
|
||||
self.bad_words_state = BadWordsState(req_states)
|
||||
self.logprob_token_ids_state = LogprobTokenIdsState(max_num_reqs, device)
|
||||
self.num_speculative_tokens = num_speculative_tokens
|
||||
self.use_flashinfer = flashinfer_sampler_supported()
|
||||
|
||||
def add_request(
|
||||
self, req_idx: int, prompt_len: int, sampling_params: SamplingParams
|
||||
@@ -77,6 +83,13 @@ class Sampler:
|
||||
# NOTE(woosuk): We intentionally compute num_nans before sampling to make clear
|
||||
# that num_nans is computed before applying penalties and temperature.
|
||||
num_nans = get_num_nans(logits) if self.compute_nans else None
|
||||
|
||||
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
|
||||
max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids(
|
||||
idx_mapping_np
|
||||
)
|
||||
return_logprobs = max_num_logprobs != NO_LOGPROBS or max_per_req_token_ids > 0
|
||||
|
||||
sampled, processed_logits = self.sample(
|
||||
logits,
|
||||
expanded_idx_mapping,
|
||||
@@ -84,13 +97,10 @@ class Sampler:
|
||||
pos,
|
||||
input_ids,
|
||||
expanded_local_pos,
|
||||
return_logprobs=return_logprobs,
|
||||
)
|
||||
|
||||
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
|
||||
max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids(
|
||||
idx_mapping_np
|
||||
)
|
||||
if max_num_logprobs != NO_LOGPROBS or max_per_req_token_ids > 0:
|
||||
if return_logprobs:
|
||||
if self.logprobs_mode == "processed_logprobs":
|
||||
logits = processed_logits
|
||||
expanded_logits = logits.shape[0] != idx_mapping_np.shape[0]
|
||||
@@ -128,6 +138,7 @@ class Sampler:
|
||||
pos: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
expanded_local_pos: torch.Tensor,
|
||||
skip_top_k_top_p: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# Copy logits to a new FP32 tensor.
|
||||
logits = torch.empty_like(logits, dtype=torch.float32).copy_(logits)
|
||||
@@ -163,6 +174,9 @@ class Sampler:
|
||||
# Apply min_p in place.
|
||||
self.sampling_states.apply_min_p(logits, expanded_idx_mapping, idx_mapping_np)
|
||||
|
||||
if skip_top_k_top_p:
|
||||
return logits
|
||||
|
||||
# Apply top_k and/or top_p. This might or might not return a new tensor.
|
||||
return self.sampling_states.apply_top_k_top_p(
|
||||
logits, expanded_idx_mapping, idx_mapping_np
|
||||
@@ -176,6 +190,7 @@ class Sampler:
|
||||
pos: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
expanded_local_pos: torch.Tensor,
|
||||
return_logprobs: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
processed_logits = self.apply_sampling_params(
|
||||
logits,
|
||||
@@ -184,16 +199,33 @@ class Sampler:
|
||||
pos,
|
||||
input_ids,
|
||||
expanded_local_pos,
|
||||
skip_top_k_top_p=True,
|
||||
)
|
||||
top_k, top_p = self.sampling_states.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np
|
||||
)
|
||||
use_flashinfer = self.use_flashinfer and not (
|
||||
# Don't use FI sampler if no requests use top_k/top_p, if there are
|
||||
# any greedy requests or per-request seeds, or if post-processed
|
||||
# logprobs need to be returned for any requests.
|
||||
(top_k is None and top_p is None)
|
||||
or (return_logprobs and self.logprobs_mode == "processed_logprobs")
|
||||
or self.sampling_states.any_greedy(idx_mapping_np)
|
||||
or self.sampling_states.any_explicit_seed(idx_mapping_np)
|
||||
)
|
||||
|
||||
# Sample the next token.
|
||||
sampled = gumbel_sample(
|
||||
processed_logits,
|
||||
expanded_idx_mapping,
|
||||
self.sampling_states.temperature.gpu,
|
||||
self.sampling_states.seeds.gpu,
|
||||
pos,
|
||||
apply_temperature=False,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
if use_flashinfer:
|
||||
sampled = flashinfer_sample(processed_logits, top_k, top_p).to(torch.int64)
|
||||
else:
|
||||
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
|
||||
sampled = gumbel_sample(
|
||||
processed_logits,
|
||||
expanded_idx_mapping,
|
||||
self.sampling_states.temperature.gpu,
|
||||
self.sampling_states.seeds.gpu,
|
||||
pos,
|
||||
apply_temperature=False,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
return sampled, processed_logits
|
||||
|
||||
@@ -24,6 +24,9 @@ class SamplingStates:
|
||||
self.top_p = UvaBackedTensor(max_num_reqs, dtype=torch.float32)
|
||||
self.min_p = UvaBackedTensor(max_num_reqs, dtype=torch.float32)
|
||||
self.seeds = UvaBackedTensor(max_num_reqs, dtype=torch.int64)
|
||||
# Tracks whether `seed` was set explicitly by the user, so callers
|
||||
# can fall back from RNG paths that don't honor per-request seeds.
|
||||
self.seeds_set = np.zeros(max_num_reqs, dtype=bool)
|
||||
|
||||
# Initialize top_k and top_p manually because 0 is an invalid value for them.
|
||||
self.top_k.np.fill(self.vocab_size)
|
||||
@@ -45,6 +48,7 @@ class SamplingStates:
|
||||
self.min_p.np[req_idx] = sampling_params.min_p
|
||||
|
||||
seed = sampling_params.seed
|
||||
self.seeds_set[req_idx] = seed is not None
|
||||
if seed is None:
|
||||
seed = np.random.randint(_NP_INT64_MIN, _NP_INT64_MAX)
|
||||
self.seeds.np[req_idx] = seed
|
||||
@@ -85,20 +89,31 @@ class SamplingStates:
|
||||
return
|
||||
apply_min_p(logits, expanded_idx_mapping, self.min_p.gpu)
|
||||
|
||||
def get_top_k_top_p(
|
||||
self, expanded_idx_mapping: torch.Tensor, idx_mapping_np: np.ndarray
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size)
|
||||
do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0)
|
||||
top_k = self.top_k.gpu[expanded_idx_mapping] if do_top_k else None
|
||||
top_p = self.top_p.gpu[expanded_idx_mapping] if do_top_p else None
|
||||
return top_k, top_p
|
||||
|
||||
def apply_top_k_top_p(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
idx_mapping_np: np.ndarray,
|
||||
) -> torch.Tensor:
|
||||
do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size)
|
||||
do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0)
|
||||
if not (do_top_k or do_top_p):
|
||||
top_k, top_p = self.get_top_k_top_p(expanded_idx_mapping, idx_mapping_np)
|
||||
if top_k is None and top_p is None:
|
||||
return logits
|
||||
|
||||
top_k = self.top_k.gpu[expanded_idx_mapping] if do_top_k else None
|
||||
top_p = self.top_p.gpu[expanded_idx_mapping] if do_top_p else None
|
||||
return apply_top_k_top_p(logits, top_k, top_p)
|
||||
|
||||
def any_greedy(self, idx_mapping_np: np.ndarray) -> bool:
|
||||
return bool(np.any(self.temperature.np[idx_mapping_np] == 0.0))
|
||||
|
||||
def any_explicit_seed(self, idx_mapping_np: np.ndarray) -> bool:
|
||||
return bool(np.any(self.seeds_set[idx_mapping_np]))
|
||||
|
||||
def max_num_logprobs(self, idx_mapping_np: np.ndarray) -> int:
|
||||
return int(np.max(self.num_logprobs[idx_mapping_np]))
|
||||
|
||||
Reference in New Issue
Block a user