diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index f98b12a379..66806ab8a9 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -19,6 +19,69 @@ if HAS_TRITON: logger = init_logger(__name__) +_FLASHINFER_MIN_VERSION = "0.2.3" + + +def flashinfer_sampler_supported() -> bool: + """Decide whether FlashInfer's top-p/top-k sampler can be used. + + Returns False (with appropriate logging) when ``VLLM_USE_FLASHINFER_SAMPLER`` + is 0, when the platform isn't CUDA, when the GPU's compute capability is + unsupported, or when the installed flashinfer is missing or too old. Raises + ``RuntimeError`` if the user explicitly opted in via the env var but + FlashInfer is unavailable. + + Note: callers must additionally ensure ``logprobs_mode`` doesn't require + post-top-k/top-p logits/logprobs for any request whose logprobs will be + returned in this step, since FlashInfer doesn't expose those. + """ + if not current_platform.is_cuda(): + return False + if not envs.VLLM_USE_FLASHINFER_SAMPLER: + logger.info_once( + "FlashInfer top-p/top-k sampling disabled via " + "VLLM_USE_FLASHINFER_SAMPLER=0." + ) + return False + from vllm.v1.attention.backends.flashinfer import FlashInferBackend + + capability = current_platform.get_device_capability() + assert capability is not None + unsupported_reason: str | None = None + if not FlashInferBackend.supports_compute_capability(capability): + unsupported_reason = ( + f"unsupported compute capability {capability.as_version_str()}" + ) + else: + try: + import flashinfer + + if version.parse(flashinfer.__version__) < version.parse( + _FLASHINFER_MIN_VERSION + ): + unsupported_reason = ( + f"flashinfer {flashinfer.__version__} is too old " + f"(>={_FLASHINFER_MIN_VERSION} required)" + ) + except ImportError: + unsupported_reason = "flashinfer is not installed" + + if unsupported_reason is None: + logger.info_once("Using FlashInfer for top-p & top-k sampling.", scope="global") + return True + if envs.is_set("VLLM_USE_FLASHINFER_SAMPLER"): + raise RuntimeError( + f"FlashInfer top-p/top-k sampling unavailable: {unsupported_reason}. " + "Unset VLLM_USE_FLASHINFER_SAMPLER=1." + ) + logger.warning_once( + "FlashInfer top-p/top-k sampling unavailable: %s; falling back. " + "Set VLLM_USE_FLASHINFER_SAMPLER=0 to silence.", + unsupported_reason, + ) + return False + + class TopKTopPSampler(nn.Module): """ Module that performs optional top-k and top-p filtering followed by @@ -30,49 +93,16 @@ class TopKTopPSampler(nn.Module): def __init__(self, logprobs_mode: LogprobsMode = "raw_logprobs") -> None: super().__init__() self.logprobs_mode = logprobs_mode - # flashinfer optimization does not apply if intermediate - # logprobs/logits after top_k/top_p need to be returned - if ( - logprobs_mode not in ("processed_logits", "processed_logprobs") - and current_platform.is_cuda() - ): - if envs.VLLM_USE_FLASHINFER_SAMPLER: - from vllm.v1.attention.backends.flashinfer import FlashInferBackend - - capability = current_platform.get_device_capability() - assert capability is not None - if FlashInferBackend.supports_compute_capability(capability): - logger.info_once( - "Using FlashInfer for top-p & top-k sampling.", - scope="global", - ) - self.forward = self.forward_cuda - elif envs.is_set("VLLM_USE_FLASHINFER_SAMPLER"): - # User explicitly opted in but the GPU can't run FlashInfer. - capability_str = capability.as_version_str() - raise RuntimeError( - "FlashInfer does not support compute capability " - f"{capability_str}, unset VLLM_USE_FLASHINFER_SAMPLER=1." - ) - else: - # Default-on path; hardware can't run FlashInfer → - # quietly fall back to the PyTorch-native sampler - # instead of failing server startup. - logger.warning_once( - "FlashInfer top-p/top-k sampling not supported on " - "compute capability %s; falling back to PyTorch-native " - "sampler. Set VLLM_USE_FLASHINFER_SAMPLER=0 to silence.", - capability.as_version_str(), - ) - self.forward = self.forward_native - else: - # User explicitly set VLLM_USE_FLASHINFER_SAMPLER=0. - logger.info_once( - "FlashInfer top-p/top-k sampling disabled via " - "VLLM_USE_FLASHINFER_SAMPLER=0; using PyTorch-native sampler." - ) - self.forward = self.forward_native - + if current_platform.is_cuda(): + # FlashInfer doesn't expose post-top-k/top-p logits/logprobs, + # so it can't be used when the configured mode requires them. + can_use_flashinfer = ( + logprobs_mode not in ("processed_logits", "processed_logprobs") + and flashinfer_sampler_supported() + ) + self.forward = ( + self.forward_cuda if can_use_flashinfer else self.forward_native + ) elif current_platform.is_cpu(): arch = current_platform.get_cpu_architecture() # Fall back to native implementation for POWERPC and RISCV. @@ -417,7 +447,7 @@ def flashinfer_sample( logits: torch.Tensor, k: torch.Tensor | None, p: torch.Tensor | None, - generators: dict[int, torch.Generator], + generators: dict[int, torch.Generator] = {}, # noqa ) -> torch.Tensor: """Sample from the logits using FlashInfer. @@ -431,11 +461,6 @@ def flashinfer_sample( """ import flashinfer - if version.parse(flashinfer.__version__) < version.parse("0.2.3"): - raise ImportError( - "FlashInfer version >= 0.2.3 required for top-k and top-p sampling. " - ) - assert not (k is None and p is None) if k is None: # Top-p only. diff --git a/vllm/v1/worker/gpu/sample/sampler.py b/vllm/v1/worker/gpu/sample/sampler.py index 8bf884fd9b..6b545aef3a 100644 --- a/vllm/v1/worker/gpu/sample/sampler.py +++ b/vllm/v1/worker/gpu/sample/sampler.py @@ -7,6 +7,11 @@ import torch import vllm.envs as envs from vllm.config.model import LogprobsMode from vllm.sampling_params import SamplingParams +from vllm.v1.sample.ops.topk_topp_sampler import ( + apply_top_k_top_p, + flashinfer_sample, + flashinfer_sampler_supported, +) from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.metrics.logits import get_num_nans from vllm.v1.worker.gpu.sample.bad_words import BadWordsState @@ -45,6 +50,7 @@ class Sampler: self.bad_words_state = BadWordsState(req_states) self.logprob_token_ids_state = LogprobTokenIdsState(max_num_reqs, device) self.num_speculative_tokens = num_speculative_tokens + self.use_flashinfer = flashinfer_sampler_supported() def add_request( self, req_idx: int, prompt_len: int, sampling_params: SamplingParams @@ -77,6 +83,13 @@ class Sampler: # NOTE(woosuk): We intentionally compute num_nans before sampling to make clear # that num_nans is computed before applying penalties and temperature. num_nans = get_num_nans(logits) if self.compute_nans else None + + max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np) + max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids( + idx_mapping_np + ) + return_logprobs = max_num_logprobs != NO_LOGPROBS or max_per_req_token_ids > 0 + sampled, processed_logits = self.sample( logits, expanded_idx_mapping, @@ -84,13 +97,10 @@ class Sampler: pos, input_ids, expanded_local_pos, + return_logprobs=return_logprobs, ) - max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np) - max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids( - idx_mapping_np - ) - if max_num_logprobs != NO_LOGPROBS or max_per_req_token_ids > 0: + if return_logprobs: if self.logprobs_mode == "processed_logprobs": logits = processed_logits expanded_logits = logits.shape[0] != idx_mapping_np.shape[0] @@ -128,6 +138,7 @@ class Sampler: pos: torch.Tensor, input_ids: torch.Tensor, expanded_local_pos: torch.Tensor, + skip_top_k_top_p: bool = False, ) -> torch.Tensor: # Copy logits to a new FP32 tensor. logits = torch.empty_like(logits, dtype=torch.float32).copy_(logits) @@ -163,6 +174,9 @@ class Sampler: # Apply min_p in place. self.sampling_states.apply_min_p(logits, expanded_idx_mapping, idx_mapping_np) + if skip_top_k_top_p: + return logits + # Apply top_k and/or top_p. This might or might not return a new tensor. return self.sampling_states.apply_top_k_top_p( logits, expanded_idx_mapping, idx_mapping_np @@ -176,6 +190,7 @@ class Sampler: pos: torch.Tensor, input_ids: torch.Tensor, expanded_local_pos: torch.Tensor, + return_logprobs: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: processed_logits = self.apply_sampling_params( logits, @@ -184,16 +199,33 @@ class Sampler: pos, input_ids, expanded_local_pos, + skip_top_k_top_p=True, + ) + top_k, top_p = self.sampling_states.get_top_k_top_p( + expanded_idx_mapping, idx_mapping_np + ) + use_flashinfer = self.use_flashinfer and not ( + # Don't use FI sampler if no requests use top_k/top_p, if there are + # any greedy requests or per-request seeds, or if post-processed + # logprobs need to be returned for any requests. + (top_k is None and top_p is None) + or (return_logprobs and self.logprobs_mode == "processed_logprobs") + or self.sampling_states.any_greedy(idx_mapping_np) + or self.sampling_states.any_explicit_seed(idx_mapping_np) ) # Sample the next token. - sampled = gumbel_sample( - processed_logits, - expanded_idx_mapping, - self.sampling_states.temperature.gpu, - self.sampling_states.seeds.gpu, - pos, - apply_temperature=False, - use_fp64=self.use_fp64_gumbel, - ) + if use_flashinfer: + sampled = flashinfer_sample(processed_logits, top_k, top_p).to(torch.int64) + else: + processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p) + sampled = gumbel_sample( + processed_logits, + expanded_idx_mapping, + self.sampling_states.temperature.gpu, + self.sampling_states.seeds.gpu, + pos, + apply_temperature=False, + use_fp64=self.use_fp64_gumbel, + ) return sampled, processed_logits diff --git a/vllm/v1/worker/gpu/sample/states.py b/vllm/v1/worker/gpu/sample/states.py index f247acba07..bf2f1ce78f 100644 --- a/vllm/v1/worker/gpu/sample/states.py +++ b/vllm/v1/worker/gpu/sample/states.py @@ -24,6 +24,9 @@ class SamplingStates: self.top_p = UvaBackedTensor(max_num_reqs, dtype=torch.float32) self.min_p = UvaBackedTensor(max_num_reqs, dtype=torch.float32) self.seeds = UvaBackedTensor(max_num_reqs, dtype=torch.int64) + # Tracks whether `seed` was set explicitly by the user, so callers + # can fall back from RNG paths that don't honor per-request seeds. + self.seeds_set = np.zeros(max_num_reqs, dtype=bool) # Initialize top_k and top_p manually because 0 is an invalid value for them. self.top_k.np.fill(self.vocab_size) @@ -45,6 +48,7 @@ class SamplingStates: self.min_p.np[req_idx] = sampling_params.min_p seed = sampling_params.seed + self.seeds_set[req_idx] = seed is not None if seed is None: seed = np.random.randint(_NP_INT64_MIN, _NP_INT64_MAX) self.seeds.np[req_idx] = seed @@ -85,20 +89,31 @@ class SamplingStates: return apply_min_p(logits, expanded_idx_mapping, self.min_p.gpu) + def get_top_k_top_p( + self, expanded_idx_mapping: torch.Tensor, idx_mapping_np: np.ndarray + ) -> tuple[torch.Tensor | None, torch.Tensor | None]: + do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size) + do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0) + top_k = self.top_k.gpu[expanded_idx_mapping] if do_top_k else None + top_p = self.top_p.gpu[expanded_idx_mapping] if do_top_p else None + return top_k, top_p + def apply_top_k_top_p( self, logits: torch.Tensor, expanded_idx_mapping: torch.Tensor, idx_mapping_np: np.ndarray, ) -> torch.Tensor: - do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size) - do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0) - if not (do_top_k or do_top_p): + top_k, top_p = self.get_top_k_top_p(expanded_idx_mapping, idx_mapping_np) + if top_k is None and top_p is None: return logits - - top_k = self.top_k.gpu[expanded_idx_mapping] if do_top_k else None - top_p = self.top_p.gpu[expanded_idx_mapping] if do_top_p else None return apply_top_k_top_p(logits, top_k, top_p) + def any_greedy(self, idx_mapping_np: np.ndarray) -> bool: + return bool(np.any(self.temperature.np[idx_mapping_np] == 0.0)) + + def any_explicit_seed(self, idx_mapping_np: np.ndarray) -> bool: + return bool(np.any(self.seeds_set[idx_mapping_np])) + def max_num_logprobs(self, idx_mapping_np: np.ndarray) -> int: return int(np.max(self.num_logprobs[idx_mapping_np]))