[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack

- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions,
  rl, render-launcher and batch entrypoints; keep scale-out token routes
- Drop the Swagger static bundle and the /docs, /redoc routes
- Delete device-keyed tuning configs for fused_moe, quant utils, mamba
  select-state and helion; no sm86 entry ever existed for any of them
- Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant
  quantizers and their config, metrics and loader plumbing
- Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu
  requirements and the setup.py cpu target path
- Delete 29 environment variables nothing reads and 14 chat-template
  fallbacks for models this fork no longer registers
- Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark,
  dsv3 gemm) with their kernels, bindings, wrappers and CMake entries
- Delete the connector test suites that targeted removed connectors
- Cut the Mistral tokenizer and parser stack, its tests, and the
  mistral_common and ijson dependency pins
- Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier
  cut deleted while keeping the model tree

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 17:10:56 +07:00
co-authored by Hinata
parent bd70a61c20
commit 2066d3143f
1086 changed files with 122 additions and 291919 deletions
-270
View File
@@ -339,17 +339,6 @@ def fused_add_rms_norm(
torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
def muse_spark_mul_tanh_soft_cap(
x: torch.Tensor,
multiplier: float,
soft_cap_temp: float,
) -> torch.Tensor:
"""Cast to FP32 and apply ``temp * tanh(x * multiplier / temp)`` in one pass."""
out = torch.empty_like(x, dtype=torch.float32)
torch.ops._C.muse_spark_mul_tanh_soft_cap(out, x, multiplier, soft_cap_temp)
return out
def fused_qk_norm_rope(
qkv: torch.Tensor,
num_heads_q: int,
@@ -2609,197 +2598,6 @@ def reshape_and_cache_flash(
)
def fused_qknorm_rope_cache(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
slot_mapping: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
num_q_heads: int,
num_kv_heads: int,
head_dim: int,
rotary_dim: int,
eps: float,
norm_weight_bias: float = 0.0,
kv_cache_dtype: str = "auto",
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fuse per-head RMSNorm, partial NeoX RoPE and KV insertion.
Norm weights may be FP32 independently of QKV. A zero rotary dimension
skips RoPE; norm outputs are rounded to QKV dtype before rotation.
"""
if rotary_dim < 0 or rotary_dim > head_dim or rotary_dim % 8:
raise ValueError("rotary_dim must be a multiple of 8 in [0, head_dim]")
q = qkv.new_empty((qkv.shape[0], num_q_heads * head_dim))
k = qkv.new_empty((qkv.shape[0], num_kv_heads * head_dim))
v = torch.empty_like(k)
cos, sin = cos_sin_cache.chunk(2, dim=-1)
torch.ops._C.fused_qknorm_rope_cache(
q,
k,
v,
qkv,
q_weight,
k_weight,
cos,
sin,
positions,
slot_mapping,
key_cache,
value_cache,
head_dim,
num_q_heads,
num_kv_heads,
rotary_dim // 2,
eps,
norm_weight_bias,
kv_cache_dtype,
k_scale,
v_scale,
)
return q, k, v
def fused_muse_spark_qknorm_rope_kv_insert(
qkv: torch.Tensor,
positions: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
eps: float,
cos_sin_cache: torch.Tensor | None = None,
rotary_dim: int = 0,
slot_mapping: torch.Tensor | None = None,
kv_cache: torch.Tensor | None = None,
block_size: int = 0,
kv_cache_dtype: str = "auto",
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
q_fp8_out: torch.Tensor | None = None,
q_scale: torch.Tensor | None = None,
) -> None:
"""In-place weightless QK RMSNorm + GPT-J RoPE, optionally writing k/v to cache.
When ``kv_cache`` is given, the caller must skip the attention layer's own KV write.
With ``q_fp8_out``, write scaled E4M3 Q there instead of rewriting Q in ``qkv``.
"""
torch.ops._C.fused_muse_spark_qknorm_rope_kv_insert(
qkv,
positions,
num_heads,
num_kv_heads,
head_dim,
eps,
cos_sin_cache,
rotary_dim,
slot_mapping,
kv_cache,
block_size,
kv_cache_dtype,
k_scale,
v_scale,
q_fp8_out,
q_scale,
)
def fused_minimax_m3_qknorm_rope_kv_insert(
qkv: torch.Tensor,
q_norm_weight: torch.Tensor,
k_norm_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
num_heads: int,
num_kv_heads: int,
rotary_dim: int,
eps: float,
index_q_norm_weight: torch.Tensor | None = None,
index_k_norm_weight: torch.Tensor | None = None,
num_index_heads: int = 0,
slot_mapping: torch.Tensor | None = None,
index_slot_mapping: torch.Tensor | None = None,
kv_cache: torch.Tensor | None = None,
index_cache: torch.Tensor | None = None,
block_size: int = 0,
q_out: torch.Tensor | None = None,
index_q_out: torch.Tensor | None = None,
kv_cache_dtype: str = "auto",
skip_index_branch: bool = False,
q_fp8_out: torch.Tensor | None = None,
q_fp8_scale: float = 1.0,
kv_k_scale: torch.Tensor | None = None,
kv_v_scale: torch.Tensor | None = None,
) -> None:
"""Fused MiniMax-M3 attention pre-processing (in-place).
Applies Gemma RMSNorm + partial NeoX RoPE to ``qkv`` in place. ``qkv`` is a
single fused tensor:
- dense layer (``num_index_heads == 0``): ``[q | k | v]``;
- sparse layer (``num_index_heads > 0``): ``[q | k | v | index_q |
index_k]`` — the index branch is read straight out of ``qkv``.
When ``kv_cache`` is given (sparse serving), also scatter-inserts the
normed/roped k & v into the paged KV cache by ``slot_mapping`` and the
index key into ``index_cache`` by ``index_slot_mapping``. ``kv_cache_dtype``
selects the cache storage/conversion path. If
``index_slot_mapping`` is omitted, ``slot_mapping`` is used for both caches.
If ``q_out`` / ``index_q_out`` (contiguous ``[N, nq*128]`` / ``[N,
niq*128]``) are given, the normed/roped q / index_q are written there
instead of in place — folding the de-interleave into this kernel's store so
callers skip a separate ``.contiguous()`` copy before the SM100 sparse
attention's flat TMA descriptor.
If ``q_fp8_out`` is given, the same normalized q is also written in FP8
E4M3 using ``q_fp8_scale`` as its dequantization scale. Without ``q_out``,
q is then written only in FP8 and the q slice of ``qkv`` is left as is.
``kv_cache_dtype="nvfp4"`` quantizes k/v into the packed HND
``[num_blocks, 2 * num_kv_heads, block_size, 72]`` uint8 cache using the
device dequantization scales ``kv_k_scale``/``kv_v_scale``. Slot
``2 * head + side`` (K = 0, V = 1) holds that head's E2M1 data followed by
its E4M3 block scales, so every head is one contiguous run of the page.
When ``skip_index_branch`` is true, sparse rows still keep their packed
``[index_q | index_k]`` tail, but the kernel only processes the main q/k/v
branches and main KV cache. This is used by MiniMax-M3 index-topk reuse
layers that consume top-k block ids selected by an earlier sparse layer.
"""
torch.ops._C.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
q_norm_weight,
k_norm_weight,
cos_sin_cache,
positions,
num_heads,
num_kv_heads,
rotary_dim,
eps,
index_q_norm_weight,
index_k_norm_weight,
num_index_heads,
slot_mapping,
index_slot_mapping,
kv_cache,
index_cache,
block_size,
q_out,
index_q_out,
kv_cache_dtype,
skip_index_branch,
q_fp8_out,
q_fp8_scale,
kv_k_scale,
kv_v_scale,
)
def fused_kda_decode(
x: torch.Tensor,
weight: torch.Tensor,
@@ -2922,35 +2720,6 @@ def concat_and_cache_mla_grouped(
)
def kimi_k3_attn_res(
prefix: torch.Tensor,
delta: torch.Tensor | None,
blocks: torch.Tensor,
norm_weight: torch.Tensor,
qk_weight: torch.Tensor,
output_norm_weight: torch.Tensor | None,
num_blocks: int,
block_write_idx: int,
eps: float,
output_norm_eps: float,
) -> torch.Tensor:
output = torch.empty_like(prefix)
torch.ops._C.kimi_k3_attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
output,
num_blocks,
block_write_idx,
eps,
output_norm_eps,
)
return output
def concat_and_cache_mla_rope_fused(
positions: torch.Tensor,
q_pe: torch.Tensor,
@@ -3435,24 +3204,6 @@ def sm100_cutlass_mla_get_workspace_size(
)
def dsv3_fused_a_gemm(
output: torch.Tensor,
mat_a: torch.Tensor,
mat_b: torch.Tensor,
enable_pdl: bool = False,
) -> None:
"""Low-latency fused-A-style GEMM (SM 9.0+, BF16, 1-16 tokens).
Computes ``output = mat_a @ mat_b`` for the compiled Kimi K3 and
DeepSeek V3 projection shapes. ``mat_a`` and ``output`` are row-major;
``mat_b`` is the column-major transposed weight. ``enable_pdl`` permits
programmatic dependent launch for callers that have validated it.
Requires SM 9.0+.
"""
torch.ops._C.dsv3_fused_a_gemm(output, mat_a, mat_b, enable_pdl)
def weight_packed_linear_cpu(
x: torch.Tensor,
packed_weight: torch.Tensor,
@@ -4841,24 +4592,3 @@ if hasattr(torch.ops._C, "hadacore_transform"):
@register_fake("_C::hadacore_transform")
def _hadacore_transform_fake(x: torch.Tensor, inplace: bool) -> torch.Tensor:
return torch.empty_like(x) if not inplace else x
if hasattr(torch.ops._C, "minimax_allreduce_rms_qk"):
@register_fake("_C::minimax_allreduce_rms_qk")
def _minimax_allreduce_rms_qk_fake(
qkv: torch.Tensor,
norm_weight_q: torch.Tensor,
norm_weight_k: torch.Tensor,
workspace: torch.Tensor,
q_size: int,
kv_size: int,
rank: int,
nranks: int,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
token_num = qkv.shape[0]
return (
torch.empty([token_num, q_size], dtype=qkv.dtype, device=qkv.device),
torch.empty([token_num, kv_size], dtype=qkv.dtype, device=qkv.device),
)
+3 -42
View File
@@ -86,16 +86,7 @@ _REGISTERED_MODEL_CLASS_OVERRIDES: set[tuple[str, str]] = set()
RunnerOption = Literal["auto", RunnerType]
ConvertType = Literal["none", "embed", "classify"]
ConvertOption = Literal["auto", ConvertType]
TokenizerMode = Literal[
"auto",
"hf",
"mistral",
"deepseek_v32",
"deepseek_v4",
"inkling",
"kimi_k3",
"cohere",
]
TokenizerMode = Literal["auto", "hf"]
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
LogprobsMode = Literal[
"raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
@@ -172,24 +163,14 @@ class ModelConfig:
tokenizer_mode: TokenizerMode | str = "auto"
"""Tokenizer mode:
- "auto" will use the tokenizer from `mistral_common` for Mistral models
if available, otherwise it will use the "hf" tokenizer.
- "auto" will use the "hf" tokenizer.
- "hf" will use the fast tokenizer if available.
- "mistral" will always use the tokenizer from `mistral_common`.
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
- "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
- "deepseek_v41" will use the DeepSeek V4.1 prompt encoder.
- "kimi_k3" will always use the "hf" tokenizer but render chat prompts
with Kimi K3's Python XTML encoding instead of a Jinja template.
- "cohere" uses the standard HF tokenizer but renders the chat template
via the `cohere_melody` library (cmd3 / cmd4 templates) instead of
Jinja, and surfaces grounded-citation metadata on responses.
- Other custom values can be supported via plugins.
To swap the Rust BPE backend that powers HF fast tokenizers for the
[fastokens](https://github.com/crusoecloud/fastokens) implementation, set
`VLLM_USE_FASTOKENS=1` instead — that override applies to any mode that
loads an HF fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, …)."""
loads an HF fast tokenizer."""
trust_remote_code: bool = False
"""Trust remote code (e.g., from HuggingFace) when downloading the model
and tokenizer."""
@@ -267,8 +248,6 @@ class ModelConfig:
(linear, moe) and ignore patterns; see :class:`QuantizationConfigArgs`.
Auto-populated from the matching online shorthand when `quantization` is
one of the values in `ONLINE_QUANT_SHORTHAND_NAMES`."""
allow_deprecated_quantization: bool = False
"""Whether to allow deprecated quantization methods."""
enforce_eager: bool = False
"""Whether to always use eager-mode PyTorch. If True, we will disable CUDA
graph and always execute the model in eager mode. If False, we will use
@@ -403,8 +382,6 @@ class ModelConfig:
logits_processors: list[str | type[LogitsProcessor]] | None = None
"""One or more logits processors' fully-qualified class names or class
definitions"""
io_processor_plugin: str | None = None
"""IOProcessor plugin name to load at model startup"""
renderer_num_workers: int = 1
"""Number of worker threads in the renderer thread pool. The pool is
consumed by the async renderer path (e.g. the OpenAI-compatible API
@@ -483,7 +460,6 @@ class ModelConfig:
"hf_token",
"hf_overrides",
"logits_processors",
"io_processor_plugin",
"pooler_config",
"multimodal_config",
"limit_mm_per_prompt",
@@ -1440,21 +1416,6 @@ class ModelConfig:
)
current_platform.verify_quantization(self.quantization)
if self.quantization in me_quant.DEPRECATED_QUANTIZATION_METHODS:
if self.allow_deprecated_quantization:
logger.warning(
"The quantization method %s is deprecated "
"and will be removed in future versions of vLLM.",
self.quantization,
)
else:
raise ValueError(
"The quantization method %s is deprecated "
"and will be removed in future versions of vLLM. To bypass, "
"set `--allow-deprecated-quantization`.",
self.quantization,
)
def _verify_cuda_graph(self) -> None:
# CUDAGraph capture not supported for encoder-decoder models on ROCm
unsupported_rocm = self.is_encoder_decoder
-192
View File
@@ -1,192 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from cutlass import BFloat16, Constexpr, Float32, Uint16, Uint32, cute
from cutlass._mlir import ir
from cutlass._mlir.dialects import llvm, vector
from cutlass.cutlass_dsl import T, dsl_user_op
@dsl_user_op
def fp32x2_to_bf16x2(a: Float32, b: Float32, *, loc=None, ip=None) -> Uint32:
out = llvm.inline_asm(
T.i32(),
[a.ir_value(loc=loc, ip=ip), b.ir_value(loc=loc, ip=ip)],
"cvt.rn.bf16x2.f32 $0, $2, $1;",
"=r,f,f",
has_side_effects=False,
is_align_stack=False,
)
return Uint32(out)
@dsl_user_op
def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]:
if isinstance(data, Uint32):
out = llvm.inline_asm(
llvm.StructType.get_literal([T.f32(), T.f32()]),
[data.ir_value(loc=loc, ip=ip)],
"shl.b32 $0, $2, 16;\n\tand.b32 $1, $2, 0xFFFF0000;",
"=f,=f,r",
has_side_effects=False,
is_align_stack=False,
loc=loc,
ip=ip,
)
return (
Float32(llvm.extractvalue(T.f32(), out, [0], loc=loc, ip=ip)),
Float32(llvm.extractvalue(T.f32(), out, [1], loc=loc, ip=ip)),
)
elif isinstance(data, (cute.Tensor, cute.TensorSSA)):
# recast to 32-bit registers if needed
if data.element_type == BFloat16:
data = cute.recast_tensor(data, Uint32)
assert data.element_type == Uint32
# NOTE: the output is always 1D
size = cute.size(data.shape)
out = cute.make_rmem_tensor(size * 2, Float32)
for i in range(size):
out[i * 2], out[i * 2 + 1] = bf16x2_to_fp32x2(data[i])
return out
else:
raise ValueError(f"Unsupported type {type(data)}")
@dsl_user_op
def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
# there is only fp8->fp16 conversion, hence we need to go
# round trip through fp16.
out = llvm.inline_asm(
llvm.StructType.get_literal([T.i32()] * 2),
[x.ir_value(loc=loc, ip=ip)],
"{\n\t"
".reg .b16 x0, x1;\n\t"
".reg .b16 t00, t01, t10, t11;\n\t"
"mov.b32 {x0, x1}, $2;\n\t"
"cvt.rn.f16x2.e4m3x2 $0, x0;\n\t"
"cvt.rn.f16x2.e4m3x2 $1, x1;\n\t"
"mov.b32 {t00, t01}, $0;\n\t"
"mov.b32 {t10, t11}, $1;\n\t"
"cvt.rn.bf16.f16 t00, t00;\n\t"
"cvt.rn.bf16.f16 t01, t01;\n\t"
"cvt.rn.bf16.f16 t10, t10;\n\t"
"cvt.rn.bf16.f16 t11, t11;\n\t"
"mov.b32 $0, {t00, t01};\n\t"
"mov.b32 $1, {t10, t11};\n\t"
"}\n",
"=r,=r,r",
has_side_effects=False,
is_align_stack=False,
)
vec = vector.from_elements(
ir.VectorType.get([2], T.i32(), loc=loc),
[llvm.extractvalue(T.i32(), out, [i], loc=loc, ip=ip) for i in range(2)],
loc=loc,
ip=ip,
)
return cute.TensorSSA(vec, 2, Uint32)
@dsl_user_op
def fp32x2_to_fp8x2(a0: Float32, a1: Float32, *, loc=None, ip=None) -> Uint16:
out = llvm.inline_asm(
T.i16(),
[
a0.ir_value(loc=loc, ip=ip),
a1.ir_value(loc=loc, ip=ip),
],
"cvt.rn.satfinite.e4m3x2.f32 $0, $2, $1;",
"=h,f,f",
has_side_effects=False,
is_align_stack=False,
)
return Uint16(out)
@dsl_user_op
def fp8x4_to_fp16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
out = llvm.inline_asm(
llvm.StructType.get_literal([T.i32()] * 2),
[x.ir_value(loc=loc, ip=ip)],
"{\n\t"
".reg .b16 lo, hi;\n\t"
"mov.b32 {lo, hi}, $2;\n\t"
"cvt.rn.f16x2.e4m3x2 $0, lo;\n\t"
"cvt.rn.f16x2.e4m3x2 $1, hi;\n\t"
"}\n",
"=r,=r,r",
has_side_effects=False,
is_align_stack=False,
loc=loc,
ip=ip,
)
vec = vector.from_elements(
ir.VectorType.get([2], T.i32(), loc=loc),
[llvm.extractvalue(T.i32(), out, [i], loc=loc, ip=ip) for i in range(2)],
loc=loc,
ip=ip,
)
return cute.TensorSSA(vec, 2, Uint32)
@dsl_user_op
def fp32x4_to_fp8x4(
a0: Float32,
a1: Float32,
a2: Float32,
a3: Float32,
*,
loc=None,
ip=None,
) -> Uint32:
# Pack four FP32 values into one b32 of four e4m3 bytes, byte order
# {a0, a1, a2, a3} from low to high address.
out = llvm.inline_asm(
T.i32(),
[
a0.ir_value(loc=loc, ip=ip),
a1.ir_value(loc=loc, ip=ip),
a2.ir_value(loc=loc, ip=ip),
a3.ir_value(loc=loc, ip=ip),
],
"{\n\t"
".reg .b16 t0, t1;\n\t"
"cvt.rn.satfinite.e4m3x2.f32 t0, $2, $1;\n\t"
"cvt.rn.satfinite.e4m3x2.f32 t1, $4, $3;\n\t"
"mov.b32 $0, {t0, t1};\n\t"
"}\n",
"=r,f,f,f,f",
has_side_effects=False,
is_align_stack=False,
)
return Uint32(out)
@dsl_user_op
def fp32x8_to_fp4x8(
vals: cute.Tensor,
offset: Constexpr[int],
*,
loc=None,
ip=None,
) -> Uint32:
# Pack eight scaled FP32 values into four E2M1x2 bytes, returned as one b32.
assert vals.element_type is Float32
out = llvm.inline_asm(
T.i32(),
[vals[offset + i].ir_value(loc=loc, ip=ip) for i in range(8)],
"{\n\t"
".reg .b8 x0, x1, x2, x3;\n\t"
"cvt.rn.satfinite.e2m1x2.f32 x0, $2, $1;\n\t"
"cvt.rn.satfinite.e2m1x2.f32 x1, $4, $3;\n\t"
"cvt.rn.satfinite.e2m1x2.f32 x2, $6, $5;\n\t"
"cvt.rn.satfinite.e2m1x2.f32 x3, $8, $7;\n\t"
"mov.b32 $0, {x0, x1, x2, x3};\n\t"
"}\n",
"=r,f,f,f,f,f,f,f,f",
has_side_effects=False,
is_align_stack=False,
)
return Uint32(out)
-47
View File
@@ -1,47 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from cutlass import Int32
from cutlass._mlir.dialects import nvvm
from cutlass.cutlass_dsl import dsl_user_op
_SPACE_MAP = {
"cta": nvvm.MBarrierSpaceKind.CTA,
"cluster": nvvm.MBarrierSpaceKind.CLUSTER,
}
_MEMORY_ORDER_MAP = {
"weak": nvvm.MemOrderKind.WEAK,
"relaxed": nvvm.MemOrderKind.RELAXED,
"acquire": nvvm.MemOrderKind.ACQUIRE,
"release": nvvm.MemOrderKind.RELEASE,
"acq_rel": nvvm.MemOrderKind.ACQ_REL,
}
@dsl_user_op
def arrive(mbar, space: str = "cta", order: str = "relaxed", *, loc=None, ip=None):
nvvm.mbarrier_txn(
mbar.to_llvm_ptr(loc=loc, ip=ip),
Int32(1).ir_value(loc=loc, ip=ip),
kind=nvvm.MBarrierTxnKind.ARRIVE,
space=_SPACE_MAP[space],
order=_MEMORY_ORDER_MAP[order],
loc=loc,
ip=ip,
)
@dsl_user_op
def arrive_expect_tx(
mbar, size, space: str = "cta", order: str = "relaxed", *, loc=None, ip=None
):
nvvm.mbarrier_txn(
mbar.to_llvm_ptr(loc=loc, ip=ip),
Int32(size).ir_value(loc=loc, ip=ip),
kind=nvvm.MBarrierTxnKind.ARRIVE_EXPECT_TX,
space=_SPACE_MAP[space],
order=_MEMORY_ORDER_MAP[order],
loc=loc,
ip=ip,
)
@@ -1,635 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import fcntl
import os
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
from functools import partial
from importlib.metadata import version
from typing import TYPE_CHECKING, Any
import torch
from packaging.version import Version
from safetensors.torch import load_file, save_file
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
SupportsHMA,
)
from vllm.distributed.parallel_state import get_tensor_model_parallel_rank
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
def extract_from_kv_cache(
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
num_tokens: int,
) -> torch.Tensor:
"""Extract data from KV cache."""
block_size = kv_cache.shape[2]
return kv_cache[slot_mapping // block_size, :, slot_mapping % block_size][
:num_tokens
]
def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
"""Load hidden states written by ExampleHiddenStatesConnector.
Blocks (without polling) until the async write is complete by
acquiring a shared flock on the companion lock file. The kernel
puts the caller to sleep until the writer releases its exclusive lock.
Args:
path: The file path returned in kv_transfer_params["hidden_states_path"].
Returns:
Dict with "hidden_states" and "token_ids" tensors.
"""
lock_path = path + ".lock"
with open(lock_path) as lf:
fcntl.flock(lf, fcntl.LOCK_SH) # sleeps until writer releases LOCK_EX
data = load_file(path, device="cpu")
return data
def cleanup_hidden_states(path: str, keep_hidden_states: bool = False) -> None:
"""Clean up hidden states file and lock file after loading.
If keep_hidden_states is True, only removes the lock file
and keeps the hidden states file.
"""
lock_path = path + ".lock"
if os.path.exists(lock_path):
os.remove(lock_path)
if not keep_hidden_states and os.path.exists(path):
os.remove(path)
@dataclass
class PendingSave:
req_id: str
filename: str
token_ids: torch.Tensor
block_ids: list[int]
@dataclass
class ExampleHiddenStatesConnectorMetadata(KVConnectorMetadata):
pending_saves: list[PendingSave] = field(default_factory=list)
# req_id → filename for newly scheduled requests — the worker pre-creates
# lock files for these so the lock exists before the client receives the
# output path.
new_req_filenames: dict[str, str] = field(default_factory=dict)
class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
"""Simple debug implementation of a HiddenStatesConnector.
Simply extracts the hidden states from the kv cache and stores them to disk.
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
"""
def get_finished_count(self) -> int:
"""Number of ``finished_sending`` notifications expected per request.
Only TP rank 0 writes hidden states to disk, so each request yields a
single notification. The default (``None``) makes ``KVOutputAggregator``
wait for every worker in the world size; since ranks 1..N-1 never submit
a copy, the request would never be reported as finished and its blocks
would be leaked forever (the scheduler then spins on
``has_finished_requests()``).
"""
return 1
@classmethod
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
"""Index of the KV cache group holding the extracted hidden states.
Located by spec type so it resolves on both scheduler and worker side.
"""
if kv_cache_config is None:
return 0
from vllm.v1.kv_cache_interface import HiddenStateCacheSpec
groups = kv_cache_config.kv_cache_groups
group_ids = [
gid
for gid, group in enumerate(groups)
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
]
if len(group_ids) == 1:
return group_ids[0]
if not group_ids and len(groups) == 1:
return 0
raise ValueError(
"Could not uniquely identify the extract-hidden-states KV cache "
f"group among {len(groups)} groups; the hidden-states layer must be "
"isolated in its own group (MLA verifiers are unsupported)."
)
@staticmethod
def _get_cache_block_size(
vllm_config: "VllmConfig",
kv_cache_config: "KVCacheConfig | None",
cache_kv_group_id: int,
) -> int:
"""Block size of the hidden-states group, read from its own spec.
cache_config.block_size is bumped to a common multiple for hybrid
verifiers; the page-aligned hidden-states group keeps a smaller one.
"""
if kv_cache_config is None:
return vllm_config.cache_config.block_size
cache_group = kv_cache_config.kv_cache_groups[cache_kv_group_id]
return cache_group.kv_cache_spec.block_size
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(
vllm_config=vllm_config,
role=role,
kv_cache_config=kv_cache_config,
)
# Read the hidden-states group and its block size from the group spec;
# cache_config.block_size is bumped (wrong) for hybrid verifiers.
self._cache_kv_group_id = self._find_cache_kv_group_id(kv_cache_config)
self._block_size = self._get_cache_block_size(
vllm_config, kv_cache_config, self._cache_kv_group_id
)
self._storage_path = self._kv_transfer_config.get_from_extra_config(
"shared_storage_path", "/tmp"
)
self.cache_layers: list[str] = [] # set by self.register_kv_caches
logger.info(self._kv_transfer_config)
logger.info("Shared storage path is %s", self._storage_path)
if Version(version("safetensors")) < Version("0.8.0"):
logger.warning(
"safetensors < 0.8.0 holds the GIL during save_file, which "
"serializes the writer thread pool and hurts throughput. "
"Upgrade to safetensors >= 0.8.0 for better performance."
)
assert self._vllm_config.speculative_config is not None, (
"ExampleHiddenStatesConnector only works when using "
"'extract_hidden_states' speculative method"
)
spec_config = self._vllm_config.speculative_config.draft_model_config.hf_config
self.num_hidden_states = len(
getattr(spec_config, "eagle_aux_hidden_state_layer_ids", [])
)
# Scheduler-side state
self._pending_saves: dict[str, PendingSave] = {}
self._request_filenames: dict[str, str] = {}
# Worker-side state (set by register_kv_caches).
self._kv_cache: torch.Tensor | None = None
# Only TP rank 0 writes hidden states to disk; other TP ranks no-op.
# Set in register_kv_caches (after distributed init).
self._is_tp_rank_zero: bool = True
# Async write infrastructure (worker-side).
# Dedicated CUDA stream for DtoH copies so they don't block
# the default stream (model forward). Thread pool for disk writes.
self._copy_stream: torch.cuda.Stream | None = None # lazy init
self._executor = ThreadPoolExecutor(
max_workers=self._kv_transfer_config.get_from_extra_config(
"num_writer_threads", 8
),
thread_name_prefix="vllm-hs-save",
)
# Whether to use a filesystem lock when writing files to shared storage.
# This is necessary for online transfer clients to avoid incomplete reads,
# but can be disabled for offline tasks that run tasks in batches to completion
self.allow_custom_save_path = self._kv_transfer_config.get_from_extra_config(
"allow_custom_save_path", False
)
if self.allow_custom_save_path:
logger.warning(
"allow_custom_save_path is enabled. API clients can write "
"hidden states to arbitrary paths on the server filesystem. "
"Only enable this with trusted clients."
)
self.use_lock = self._kv_transfer_config.get_from_extra_config(
"use_synchronization_lock", True
)
# req_id → open fd on the .lock file with LOCK_EX held.
# Pre-created in wait_for_save when a request first arrives,
# consumed by _submit_async_write which passes the fd to the
# thread pool worker for release after writing.
self._lock_fds: dict[str, int] = {}
# req_id → in-flight disk-write Future for that req_id.
self._req_futures: dict[str, Future] = {}
# req_id → CUDA event marking completion of the DtoH copy. Once
# this event is complete the request is considered "done sending"
# by get_finished; clients block on the per-file flock to wait for
# the disk write itself.
self._req_copy_events: dict[str, torch.cuda.Event] = {}
# req_ids reported as finished-generating by the scheduler,
# accumulated across get_finished calls.
self._accumulated_finished_req_ids: set[str] = set()
def _get_copy_stream(self) -> torch.cuda.Stream:
"""Lazily create the copy stream (CUDA must be initialized)."""
if self._copy_stream is None:
self._copy_stream = torch.cuda.Stream()
return self._copy_stream
# ==============================
# Worker-side methods
# ==============================
def start_load_kv(self, *args, **kwargs: Any) -> None:
pass # Store-only connector — nothing to load
def wait_for_layer_load(self, layer_name: str) -> None:
pass # Store-only connector — nothing to load
def wait_for_save(self) -> None:
"""Pre-create lock files for newly arrived requests.
This runs on the worker BEFORE the scheduler returns the output
path to the client, guaranteeing that the lock file exists (and
LOCK_EX is held) by the time the client tries to open it.
"""
if not self._is_tp_rank_zero:
return
if not self.use_lock or not self.has_connector_metadata():
return
metadata = self._get_connector_metadata()
if not isinstance(metadata, ExampleHiddenStatesConnectorMetadata):
return
for req_id, filename in metadata.new_req_filenames.items():
if req_id in self._lock_fds:
continue
lock_path = filename + ".lock"
os.makedirs(os.path.dirname(lock_path), exist_ok=True)
lock_fd = os.open(lock_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644)
fcntl.flock(lock_fd, fcntl.LOCK_EX)
self._lock_fds[req_id] = lock_fd
def _on_write_done(self, req_id: str, future: Future) -> None:
"""Surface any exception from the disk-write thread and drop the
completed future from the in-flight tracking dict."""
self._req_futures.pop(req_id, None)
exc = future.exception()
if exc is not None:
logger.error("Hidden-states write failed for req_id=%s: %r", req_id, exc)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
# Delay tp rank0 initialization until after distributed init
self._is_tp_rank_zero = get_tensor_model_parallel_rank() == 0
from vllm.model_executor.models.extract_hidden_states import (
CacheOnlyAttentionLayer,
)
# Filter layers to only include CacheOnlyAttentionLayers
layers = get_layers_from_vllm_config(
self._vllm_config, CacheOnlyAttentionLayer, list(kv_caches.keys())
)
self.cache_layers = list(layers.keys())
assert len(self.cache_layers) == 1, (
f"Expected 1 CacheOnlyAttentionLayer, got {len(self.cache_layers)}"
)
self._kv_cache = kv_caches[self.cache_layers[0]]
# Block size must match the indexed buffer, else reads hit the wrong
# slots. Raise (not assert) so the check survives `python -O`.
# Views are [num_blocks, num_heads, block_size, head_size], matching what
# extract_from_kv_cache() indexes.
if self._block_size != self._kv_cache.shape[2]:
raise ValueError(
f"Hidden-states block-size mismatch: derived {self._block_size} "
f"but buffer block size is {self._kv_cache.shape[2]}; read slots "
"would be wrong (likely a hybrid block-size resolution bug)."
)
@staticmethod
def _write_tensors(
tensors: dict[str, torch.Tensor],
event: torch.cuda.Event,
filename: str,
lock_fd: int | None,
) -> None:
"""Thread worker: wait for async DtoH copy, write to disk, release lock.
``lock_fd`` is an open file descriptor on the companion ``.lock``
file with ``LOCK_EX`` already held. Closing it releases the lock,
which unblocks any client sleeping on ``LOCK_SH``.
"""
try:
event.synchronize()
save_file(tensors, filename)
finally:
if lock_fd is not None:
os.close(lock_fd) # releases LOCK_EX
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
# Hidden states are already cached by CacheOnlyAttentionLayer during
# forward. Extraction happens in get_finished once all tokens are done.
pass
def _submit_async_write(
self,
pending: PendingSave,
) -> None:
"""Extract hidden states from KV cache and submit async DtoH + disk write.
Called from get_finished for each request that has finished generating.
"""
if not self._is_tp_rank_zero:
return
assert self._kv_cache is not None
# Compute slot mapping from block_ids
block_ids_t = torch.tensor(pending.block_ids, dtype=torch.long)
num_blocks = block_ids_t.shape[0]
block_offsets = torch.arange(0, self._block_size, dtype=torch.long)
slot_mapping = (
block_offsets.reshape((1, self._block_size))
+ block_ids_t.reshape((num_blocks, 1)) * self._block_size
)
slot_mapping = slot_mapping.flatten()
num_tokens = pending.token_ids.shape[0]
copy_stream = self._get_copy_stream()
# Ensure the copy stream sees all prior writes on the default stream.
ready_event = torch.cuda.Event()
ready_event.record()
copy_stream.wait_event(ready_event)
with torch.cuda.stream(copy_stream):
# Move the CPU slot_mapping to GPU on the copy stream so the
# implicit H2D inside fancy indexing doesn't sync the default
# stream. Deliberate bulk transfer in this reference connector.
with gpu_sync_allowed():
slot_mapping_gpu = slot_mapping.to(
device=self._kv_cache.device, non_blocking=True
)
hidden_states_gpu = extract_from_kv_cache(
self._kv_cache, slot_mapping_gpu, num_tokens
)
# Async DtoH copy into pinned host memory.
pinned_hs = torch.empty_like(
hidden_states_gpu, device="cpu", pin_memory=True
)
pinned_hs.copy_(hidden_states_gpu, non_blocking=True)
# Record completion of this copy on the copy stream.
copy_done = torch.cuda.Event()
copy_done.record(copy_stream)
# token_ids is already on CPU (created in request_finished).
assert not pending.token_ids.is_cuda, (
"Expected token_ids on CPU, got CUDA tensor"
)
tensors = {
"hidden_states": pinned_hs,
"token_ids": pending.token_ids.clone(),
}
# Submit to thread pool for disk write.
prior = self._req_futures.get(pending.req_id)
assert prior is None, "Found another KV transfer request with same req_id!"
os.makedirs(os.path.dirname(pending.filename), exist_ok=True)
# Use the pre-created lock fd from wait_for_save (already holds
# LOCK_EX). Falls back to creating one here if use_lock is True
# but no pre-created fd exists (shouldn't happen in normal flow).
lock_fd = self._lock_fds.pop(pending.req_id, None)
if lock_fd is None and self.use_lock:
lock_path = pending.filename + ".lock"
lock_fd = os.open(lock_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644)
fcntl.flock(lock_fd, fcntl.LOCK_EX)
future = self._executor.submit(
self._write_tensors, tensors, copy_done, pending.filename, lock_fd
)
self._req_copy_events[pending.req_id] = copy_done
self._req_futures[pending.req_id] = future
future.add_done_callback(partial(self._on_write_done, pending.req_id))
# ==============================
# Scheduler-side methods
# ==============================
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
# This connector is store-only, so we don't need to load any tokens
return 0, False
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
# Usually used to handle allocation of new blocks for requests that are loading
# tokens from connector's external kv cache. We never load from external cache
# so this is a no-op.
assert num_external_tokens == 0, "This connector is store-only"
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify any fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
meta = ExampleHiddenStatesConnectorMetadata()
# Transfer pending saves into metadata (scheduler → worker bridge)
meta.pending_saves = list(self._pending_saves.values())
self._pending_saves.clear()
# Resolve save paths for new requests and tell the worker so it can
# pre-create lock files before the client receives the output path.
for new_req in scheduler_output.scheduled_new_reqs:
default_path = os.path.join(
self._storage_path, f"{new_req.req_id}.safetensors"
)
kv_params = (
new_req.sampling_params.extra_args.get("kv_transfer_params")
if new_req.sampling_params and new_req.sampling_params.extra_args
else None
) or {}
custom_path = kv_params.get("hidden_states_path")
if custom_path is not None and not self.allow_custom_save_path:
logger.warning(
"Request %s provided hidden_states_path but "
"allow_custom_save_path is disabled. Ignoring "
"custom path and using default.",
new_req.req_id,
)
custom_path = None
filename = custom_path or default_path
self._request_filenames[new_req.req_id] = filename
meta.new_req_filenames[new_req.req_id] = filename
return meta
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""Called exactly once when a request has finished, before its blocks are
freed.
Returns True to delay block freeing until get_finished extracts
the hidden states from the KV cache.
"""
req_id = request.request_id
filename = self._request_filenames.pop(req_id, None)
if filename is None:
# Request was aborted before it was ever scheduled, so
# build_connector_meta never recorded a filename for it. There are
# no hidden states to save and no blocks worth delaying.
self._pending_saves.pop(req_id, None)
return False, None
kv_params = request.kv_transfer_params or {}
if kv_params.get("include_output_tokens", False):
# Exclude the final token — it was the model's output, never an
# input to a forward pass, so its hidden state is not in the cache.
token_ids = torch.tensor(list(request.all_token_ids)[:-1])
elif request.prompt_token_ids is not None:
token_ids = torch.tensor(request.prompt_token_ids)
else:
logger.warning(
"Request %s has no prompt_token_ids (prompt_embeds only). "
"Saved token_ids will be empty.",
req_id,
)
token_ids = torch.tensor([], dtype=torch.long)
self._pending_saves[req_id] = PendingSave(
req_id=req_id,
filename=filename,
token_ids=token_ids,
block_ids=list(block_ids),
)
return True, {"hidden_states_path": filename}
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Extract hidden states and poll DtoH-copy completion.
On the worker side, connector metadata carries pending saves from the
scheduler. For each one we extract from the KV cache and launch an
async DtoH copy + thread-pool disk write.
We then poll accumulated finished req_ids: a request is "done sending"
once its DtoH copy event is complete. The subsequent disk write may
still be in flight; clients block on the per-file flock to wait for it.
"""
# Extract and submit async writes for newly finished requests
if self.has_connector_metadata():
connector_metadata = self._get_connector_metadata()
if isinstance(connector_metadata, ExampleHiddenStatesConnectorMetadata):
for pending in connector_metadata.pending_saves:
self._submit_async_write(pending)
# Poll for completed DtoH copies
self._accumulated_finished_req_ids.update(finished_req_ids)
done_sending: set[str] = set()
for req_id in list(self._accumulated_finished_req_ids):
event = self._req_copy_events.get(req_id)
if event is not None and not event.query():
continue # DtoH copy still in flight
self._req_copy_events.pop(req_id, None)
self._accumulated_finished_req_ids.discard(req_id)
# Only report requests we actually tracked for sending.
if event is not None:
done_sending.add(req_id)
# Clean up any leftover lock fds (e.g. aborted requests that never
# went through _submit_async_write).
lock_fd = self._lock_fds.pop(req_id, None)
if lock_fd is not None:
os.close(lock_fd)
return done_sending or None, None
def request_finished_all_groups(
self,
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
if not 0 <= self._cache_kv_group_id < len(block_ids):
# No per-group block table (e.g. a request that was never
# scheduled); treat as a no-op instead of indexing out of bounds.
return False, None
return self.request_finished(request, block_ids[self._cache_kv_group_id])
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
if cls is KVConnectorBase_V1:
raise TypeError(
"get_required_kvcache_layout should not be called "
"on the abstract base class"
)
# LBNHC means we have (num_tokens, num_heads)
# LBHNC means we have (num_heads, num_tokens)
# For now, we only support LBNHC layout since this keeps the
# hidden states for each token together in memory.
# LBHNC is primarily used when sharding heads across devices.
return "LBNHC"
-11
View File
@@ -638,7 +638,6 @@ class EngineArgs:
:class:`QuantizationConfigArgs`. Auto-populated from the matching online
shorthand when `quantization` is one of the values in
`ONLINE_QUANT_SHORTHAND_NAMES`."""
allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
enforce_eager: bool = ModelConfig.enforce_eager
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
enable_shm_tensor_arena: bool = ParallelConfig.enable_shm_tensor_arena
@@ -675,7 +674,6 @@ class EngineArgs:
mm_encoder_fp8_scale_save_margin: float = (
MultiModalConfig.mm_encoder_fp8_scale_save_margin
)
io_processor_plugin: str | None = None
renderer_num_workers: int = 1
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
@@ -981,10 +979,6 @@ class EngineArgs:
model_group.add_argument(
"--quantization-config", **model_kwargs["quantization_config"]
)
model_group.add_argument(
"--allow-deprecated-quantization",
**model_kwargs["allow_deprecated_quantization"],
)
model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
model_group.add_argument(
"--enable-return-routed-experts",
@@ -1053,9 +1047,6 @@ class EngineArgs:
model_group.add_argument(
"--logits-processors", **model_kwargs["logits_processors"]
)
model_group.add_argument(
"--io-processor-plugin", **model_kwargs["io_processor_plugin"]
)
model_group.add_argument(
"--renderer-num-workers",
**model_kwargs["renderer_num_workers"],
@@ -2000,7 +1991,6 @@ class EngineArgs:
max_model_len=self.max_model_len,
quantization=self.quantization,
quantization_config=self.quantization_config,
allow_deprecated_quantization=self.allow_deprecated_quantization,
enforce_eager=self.enforce_eager,
return_sampling_mask=self.return_sampling_mask,
max_logprobs=self.max_logprobs,
@@ -2046,7 +2036,6 @@ class EngineArgs:
mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
mm_device_do_normalize=self.mm_device_do_normalize,
mm_processor_device=self.mm_processor_device,
io_processor_plugin=self.io_processor_plugin,
renderer_num_workers=self.renderer_num_workers,
)
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-123
View File
@@ -1,123 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.entrypoints.anthropic.protocol import (
AnthropicCountTokensRequest,
AnthropicCountTokensResponse,
AnthropicError,
AnthropicErrorResponse,
AnthropicMessagesRequest,
AnthropicMessagesResponse,
)
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
router = APIRouter()
def messages(request: Request) -> AnthropicServingMessages:
return request.app.state.anthropic_serving_messages
def translate_error_response(response: ErrorResponse) -> JSONResponse:
anthropic_error = AnthropicErrorResponse(
error=AnthropicError(
type=response.error.type,
message=response.error.message,
)
)
return JSONResponse(
status_code=response.error.code, content=anthropic_error.model_dump()
)
@router.post(
"/v1/messages",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
handler = messages(raw_request)
if handler is None:
base_server = raw_request.app.state.serving_tokenization
error = base_server.create_error_response(
NotImplementedError("The model does not support Messages API")
)
return translate_error_response(error)
try:
generator = await handler.create_messages(request, raw_request)
except Exception as e:
logger.exception("Error in create_messages: %s", e)
return translate_error_response(create_error_response(e))
if isinstance(generator, ErrorResponse):
return translate_error_response(generator)
elif isinstance(generator, AnthropicMessagesResponse):
resp = generator.model_dump(exclude_none=True)
logger.debug("Anthropic Messages Response: %s", resp)
return JSONResponse(content=resp)
return StreamingResponse(content=generator, media_type="text/event-stream")
@router.post(
"/v1/messages/count_tokens",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"model": AnthropicCountTokensResponse},
HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def count_tokens(request: AnthropicCountTokensRequest, raw_request: Request):
handler = messages(raw_request)
if handler is None:
base_server = raw_request.app.state.serving_tokenization
error = base_server.create_error_response(
NotImplementedError("The model does not support Messages API")
)
return translate_error_response(error)
try:
response = await handler.count_tokens(request, raw_request)
except Exception as e:
logger.exception("Error in count_tokens: %s", e)
return translate_error_response(create_error_response(e))
if isinstance(response, ErrorResponse):
return translate_error_response(response)
return JSONResponse(content=response.model_dump(exclude_none=True))
def attach_router(app: FastAPI):
app.include_router(router)
-451
View File
@@ -1,451 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pydantic models for Anthropic API protocol."""
import time
from typing import Annotated, Any, Literal
from pydantic import BaseModel, Field, field_validator, model_validator
import vllm.envs as envs
from vllm.entrypoints.generate.base.protocol import (
MMPayload,
resolve_return_mm_kwargs_alias,
)
class AnthropicError(BaseModel):
"""Error structure for Anthropic API."""
type: str
message: str
class AnthropicErrorResponse(BaseModel):
"""Error response structure for Anthropic API."""
type: Literal["error"] = "error"
error: AnthropicError
class AnthropicUsage(BaseModel):
"""Token usage information."""
input_tokens: int
output_tokens: int
cache_creation_input_tokens: int | None = None
cache_read_input_tokens: int | None = None
class AnthropicContentBlock(BaseModel):
"""Content block in message."""
type: Literal[
"text",
"image",
"tool_use",
"tool_result",
"tool_reference",
"thinking",
"redacted_thinking",
"tool_addition",
"tool_removal",
]
text: str | None = None
# For image content
source: dict[str, Any] | None = None
# For tool use/result
id: str | None = None
tool_use_id: str | None = None
name: str | None = None
input: dict[str, Any] | None = None
content: str | list[dict[str, Any]] | None = None
is_error: bool | None = None
# For tool_reference content
tool_name: str | None = None
# For thinking content
thinking: str | None = None
signature: str | None = None
# For redacted thinking content (safety-filtered by the API)
data: str | None = None
# For tool_addition/tool_removal content
tool: "AnthropicToolChange | None" = None
@model_validator(mode="after")
def validate_tool_change(self) -> "AnthropicContentBlock":
if self.type not in ("tool_addition", "tool_removal"):
return self
if self.tool is None:
raise ValueError(f"tool is required for {self.type} blocks")
if self.type == "tool_removal" and self.tool.type != "tool_reference":
raise ValueError("tool_removal only accepts a tool_reference")
return self
def _reject_tool_changes(content: str | list[AnthropicContentBlock] | None) -> None:
if content is None or isinstance(content, str):
return
for block in content:
if block.type in ("tool_addition", "tool_removal"):
raise ValueError(
f'{block.type} blocks are only allowed in messages with role "system"'
)
class AnthropicMessage(BaseModel):
"""Message structure."""
role: Literal["user", "assistant", "system"]
content: str | list[AnthropicContentBlock]
@model_validator(mode="after")
def validate_tool_change_role(self) -> "AnthropicMessage":
if self.role != "system":
_reject_tool_changes(self.content)
return self
class AnthropicTool(BaseModel):
"""Tool definition."""
name: str
description: str | None = None
input_schema: dict[str, Any]
strict: bool | None = None
defer_loading: bool | None = None
@field_validator("input_schema")
@classmethod
def validate_input_schema(cls, v):
if not isinstance(v, dict):
raise ValueError("input_schema must be a dictionary")
if "type" not in v:
v["type"] = "object" # Default to object type
return v
class AnthropicToolChangeReference(BaseModel):
"""A tool named by a tool_addition or tool_removal block."""
type: Literal["tool_reference"]
name: str
class AnthropicToolChangeDefinition(BaseModel):
"""A tool defined by value in a tool_addition block."""
type: Literal["tool_definition"]
definition: AnthropicTool
AnthropicToolChange = Annotated[
AnthropicToolChangeReference | AnthropicToolChangeDefinition,
Field(discriminator="type"),
]
class AnthropicToolChoice(BaseModel):
"""Tool Choice definition."""
type: Literal["auto", "any", "tool", "none"]
name: str | None = None
disable_parallel_tool_use: bool | None = None
@model_validator(mode="after")
def validate_name_required_for_tool(self) -> "AnthropicToolChoice":
if self.type == "tool" and not self.name:
raise ValueError("tool_choice.name is required when type is 'tool'")
return self
class AnthropicJsonOutputFormat(BaseModel):
"""JSON output format configuration."""
json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
type: Literal["json_schema"] = "json_schema"
AnthropicEffort = Literal["low", "medium", "high", "xhigh", "max"]
# reasoning_effort sent to the chat template for thinking.type=disabled.
AnthropicDisabledThinkingEffort = Literal["none", AnthropicEffort]
AnthropicDisabledThinkingEffortOption = Literal["auto", AnthropicDisabledThinkingEffort]
AnthropicThinkingDisplay = Literal["summarized", "omitted", "updates"]
class AnthropicOutputConfig(BaseModel):
"""Configuration options for the model's output, such as the output format."""
effort: AnthropicEffort | None = None
format: AnthropicJsonOutputFormat | None = None
class AnthropicThinkingConfigEnabled(BaseModel):
"""Extended thinking with a fixed token budget.
``display`` is accepted but ignored: reasoning is always returned.
"""
type: Literal["enabled"]
budget_tokens: int = Field(ge=1024)
display: AnthropicThinkingDisplay | None = None
class AnthropicThinkingConfigDisabled(BaseModel):
type: Literal["disabled"]
class AnthropicThinkingConfigAdaptive(BaseModel):
"""Extended thinking whose depth the model chooses.
``display`` is accepted but ignored: reasoning is always returned.
"""
type: Literal["adaptive"]
display: AnthropicThinkingDisplay | None = None
AnthropicThinkingConfig = Annotated[
AnthropicThinkingConfigEnabled
| AnthropicThinkingConfigDisabled
| AnthropicThinkingConfigAdaptive,
Field(discriminator="type"),
]
class AnthropicMessagesRequest(BaseModel):
"""Anthropic Messages API request."""
model: str
messages: list[AnthropicMessage]
max_tokens: int
metadata: dict[str, Any] | None = None
output_config: AnthropicOutputConfig | None = None
thinking: AnthropicThinkingConfig | None = None
stop_sequences: (
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
) = None
stream: bool | None = False
system: str | list[AnthropicContentBlock] | None = None
temperature: float | None = None
tool_choice: AnthropicToolChoice | None = None
tools: list[AnthropicTool] | None = None
top_k: int | None = None
top_p: float | None = None
# vLLM-specific fields that are not in Anthropic spec
mm_payload: MMPayload = Field(
default="tensors",
description=(
"What the render response's `features` carry for multimodal "
"items. 'tensors' serializes the processed tensors into "
"`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
"callers that need only the token layout and item hashes, such as "
"cache-aware routers. Do not send a 'none' response to "
"`/inference/v1/generate` which reads a null `kwargs_data` as "
"every item being cached. Only supported on the render endpoints. "
"Ignored on regular generation endpoints."
),
)
return_mm_kwargs: bool | None = Field(
default=None,
deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
description=(
"Deprecated alias: false is the same as `mm_payload='none'`. "
"Ignored when `mm_payload` is set."
),
)
cache_salt: str | None = Field(
default=None,
min_length=1,
max_length=1024,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
watermarking: bool | None = Field(
default=None,
description=(
"Whether to apply the engine's configured watermark to this request."
),
)
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
default=None,
description=(
"Additional request parameters with (list of) string or "
"numeric values, used by custom extensions."
),
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model")
@classmethod
def validate_model(cls, v):
if not v:
raise ValueError("Model is required")
return v
@field_validator("max_tokens")
@classmethod
def validate_max_tokens(cls, v):
if v <= 0:
raise ValueError("max_tokens must be positive")
return v
@field_validator("system")
@classmethod
def validate_system(cls, v):
_reject_tool_changes(v)
return v
@model_validator(mode="before")
@classmethod
def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
return resolve_return_mm_kwargs_alias(data)
@model_validator(mode="after")
def validate_thinking_budget(self) -> "AnthropicMessagesRequest":
# P/D prefill legs are sent with max_tokens=1 and never decode; the
# decode leg carries the client's max_tokens and is still checked.
if self.kv_transfer_params and self.kv_transfer_params.get("do_remote_decode"):
return self
if (
isinstance(self.thinking, AnthropicThinkingConfigEnabled)
and self.thinking.budget_tokens >= self.max_tokens
):
raise ValueError("thinking.budget_tokens must be less than max_tokens")
return self
class AnthropicDelta(BaseModel):
"""Delta for streaming responses."""
type: (
Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"]
| None
) = None
text: str | None = None
thinking: str | None = None
partial_json: str | None = None
signature: str | None = None
# Message delta
stop_reason: (
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
) = None
stop_sequence: str | None = None
class AnthropicStreamEvent(BaseModel):
"""Streaming event."""
type: Literal[
"message_start",
"message_delta",
"message_stop",
"content_block_start",
"content_block_delta",
"content_block_stop",
"ping",
"error",
]
message: "AnthropicMessagesResponse | None" = None
delta: AnthropicDelta | None = None
content_block: AnthropicContentBlock | None = None
index: int | None = None
error: AnthropicError | None = None
usage: AnthropicUsage | None = None
class AnthropicMessagesResponse(BaseModel):
"""Anthropic Messages API response."""
id: str
type: Literal["message"] = "message"
role: Literal["assistant"] = "assistant"
content: list[AnthropicContentBlock]
model: str
stop_reason: (
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
) = None
stop_sequence: str | None = None
usage: AnthropicUsage | None = None
# vLLM-specific fields that are not in Anthropic spec
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None, description="ECTransfer parameters."
)
def model_post_init(self, __context):
if not self.id:
self.id = f"msg_{int(time.time() * 1000)}"
class AnthropicContextManagement(BaseModel):
"""Context management information for token counting."""
original_input_tokens: int
class AnthropicCountTokensRequest(BaseModel):
"""Anthropic messages.count_tokens request."""
model: str
messages: list[AnthropicMessage]
output_config: AnthropicOutputConfig | None = None
system: str | list[AnthropicContentBlock] | None = None
thinking: AnthropicThinkingConfig | None = None
tool_choice: AnthropicToolChoice | None = None
tools: list[AnthropicTool] | None = None
# vLLM-specific fields that are not in Anthropic spec
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model")
@classmethod
def validate_model(cls, v):
if not v:
raise ValueError("Model is required")
return v
@field_validator("system")
@classmethod
def validate_system(cls, v):
_reject_tool_changes(v)
return v
class AnthropicCountTokensResponse(BaseModel):
"""Anthropic messages.count_tokens response."""
input_tokens: int
context_management: AnthropicContextManagement | None = None
File diff suppressed because it is too large Load Diff
-115
View File
@@ -1,115 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import inspect
import uvloop
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.launchers.cli_args import (
make_arg_parser,
validate_parsed_serve_args,
)
from vllm.entrypoints.launchers.render.entry import run_launch_fastapi
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = init_logger(__name__)
DESCRIPTION = "Launch individual vLLM components."
class LaunchSubcommandBase(CLISubcommand):
"""The base class of subcommands for `vllm launch`."""
help: str
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
"""Add the CLI arguments to the parser.
By default, uses the subcommand's docstring as the description and adds
the standard vLLM serving arguments.
Subclasses can override to add component-specific arguments.
"""
if cls.__doc__:
parser.description = inspect.cleandoc(cls.__doc__)
make_arg_parser(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
raise NotImplementedError
class RenderSubcommand(LaunchSubcommandBase):
"""`vllm launch render` starts a GPU-less rendering server for preprocessing
and postprocessing only.
```bash
vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100
```
This command reuses the standard serving parser, so model, frontend,
networking, and related CLI options follow the same conventions as
[`vllm serve`](https://docs.vllm.ai/en/latest/cli/serve/).
"""
name = "render"
help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."
@staticmethod
def cmd(args: argparse.Namespace) -> None:
uvloop.run(run_launch_fastapi(args))
class LaunchSubcommand(CLISubcommand):
"""The `launch` subcommand for the vLLM CLI.
Uses nested sub-subcommands so each component can define its own
arguments independently (e.g. ``vllm launch render``).
"""
name = "launch"
@staticmethod
def cmd(args: argparse.Namespace) -> None:
if hasattr(args, "model_tag") and args.model_tag is not None:
args.model = args.model_tag
args.launch_command(args)
def validate(self, args: argparse.Namespace) -> None:
validate_parsed_serve_args(args)
def subparser_init(
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
launch_parser = subparsers.add_parser(
self.name,
help=DESCRIPTION,
description=DESCRIPTION,
usage=f"vllm {self.name} <component> [options]",
)
launch_subparsers = launch_parser.add_subparsers(
required=True, dest="launch_component"
)
for cmd_cls in LaunchSubcommandBase.__subclasses__():
cmd_subparser = launch_subparsers.add_parser(
cmd_cls.name,
help=cmd_cls.help,
usage=f"vllm {self.name} {cmd_cls.name} [options]",
)
cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)
cmd_cls.add_cli_args(cmd_subparser)
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
subcmd=f"{self.name} {cmd_cls.name}"
)
return launch_parser
def cmd_init() -> list[CLISubcommand]:
return [LaunchSubcommand()]
-4
View File
@@ -42,10 +42,8 @@ def main():
import vllm.entrypoints.cli.collect_env
import vllm.entrypoints.cli.download_kernels
import vllm.entrypoints.cli.launch
import vllm.entrypoints.cli.openai
import vllm.entrypoints.cli.preload
import vllm.entrypoints.cli.run_batch
import vllm.entrypoints.cli.serve
from vllm.entrypoints.serve.utils.api_utils import (
VLLM_SUBCMD_PARSER_EPILOG,
@@ -56,11 +54,9 @@ def main():
CMD_MODULES = [
vllm.entrypoints.cli.openai,
vllm.entrypoints.cli.serve,
vllm.entrypoints.cli.launch,
vllm.entrypoints.cli.collect_env,
vllm.entrypoints.cli.download_kernels,
vllm.entrypoints.cli.preload,
vllm.entrypoints.cli.run_batch,
]
cli_env_setup()
-68
View File
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import asyncio
import importlib.metadata
import typing
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
if typing.TYPE_CHECKING:
from vllm.utils.argparse_utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
logger = init_logger(__name__)
class RunBatchSubcommand(CLISubcommand):
"""The `run-batch` subcommand for vLLM CLI."""
name = "run-batch"
@staticmethod
def cmd(args: argparse.Namespace) -> None:
from vllm.entrypoints.launchers.run_batch import main as run_batch_main
logger.info(
"vLLM batch processing API version %s", importlib.metadata.version("vllm")
)
logger.info("args: %s", args)
# Start the Prometheus metrics server.
# LLMEngine uses the Prometheus client
# to publish metrics at the /metrics endpoint.
if args.enable_metrics:
from prometheus_client import start_http_server
logger.info("Prometheus metrics enabled")
start_http_server(port=args.port, addr=args.url)
else:
logger.info("Prometheus metrics disabled")
asyncio.run(run_batch_main(args))
def subparser_init(
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
from vllm.entrypoints.launchers.run_batch import make_arg_parser
run_batch_parser = subparsers.add_parser(
self.name,
help="Run batch prompts and write results to file.",
description=(
"Run batch prompts using vLLM's OpenAI-compatible API.\n"
"Supports local or HTTP input/output files."
),
usage="vllm run-batch -i INPUT.jsonl -o OUTPUT.jsonl --model <model>",
)
run_batch_parser = make_arg_parser(run_batch_parser)
run_batch_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
return run_batch_parser
def cmd_init() -> list[CLISubcommand]:
return [RunBatchSubcommand()]
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-309
View File
@@ -1,309 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FastAPI router for the Cohere Chat v2 API.
Exposes two routes:
* ``POST /cohere/v2/chat`` - the chat endpoint itself.
* ``POST /cohere/v2/chat/render`` - tokenize a request without running
generation, mirroring ``POST /v1/chat/completions/render``.
The Cohere v2 protocol models are sourced from the official ``cohere``
Python SDK (``pip install cohere``). To keep that an *optional*
dependency for vLLM, the SDK-dependent imports - and the route handlers
themselves - are gated on a one-shot probe at module load. If the SDK
isn't installed, :func:`attach_router` becomes a no-op (with an info
log) and vLLM continues to boot normally.
Even when the SDK is installed, :func:`attach_router` also requires
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
the chat route. The render route additionally requires scale-out
endpoints to be enabled, i.e. ``--enable-scale-out`` or
``--tokens-only``. These gates keep unrelated deployments from
accidentally exposing the APIs.
Note: the handlers must live at module scope (not inside
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
the ``CohereChatV2Request`` body annotation against the module's
globals. Defining them locally inside ``attach_router`` would hide the
type from ``get_type_hints``, causing FastAPI to silently degrade the
body parameter into a query parameter and reject every request with
422.
"""
import json
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.middleware.base import BaseHTTPMiddleware
import vllm.envs as envs
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.logger import init_logger
_COHERE_PATH_PREFIX = "/cohere/"
logger = init_logger(__name__)
try:
import cohere # noqa: F401 -- dependency probe
except ImportError:
_SDK_AVAILABLE = False
else:
_SDK_AVAILABLE = True
if _SDK_AVAILABLE:
from vllm.entrypoints.cohere.protocol import (
CohereChatV2Request,
CohereChatV2Response,
CohereError,
)
from vllm.entrypoints.cohere.serving import CohereServingChatV2
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
router = APIRouter()
render_router = APIRouter()
def _serving(request: Request) -> CohereServingChatV2 | None:
return getattr(request.app.state, "cohere_serving_chat_v2", None)
def _serving_render(request: Request) -> ServingRender | None:
return getattr(request.app.state, "serving_render", None)
def _request_id(raw_request: Request | None) -> str | None:
"""Best-effort lookup of the active request id.
Prefers the id the underlying chat handler stamped onto
``raw_request.state.request_metadata`` (if it got that far before
failing), falling back to the ``X-Request-Id`` HTTP header. May
return ``None`` if neither is available, in which case the field
is omitted from the response.
"""
if raw_request is None:
return None
meta = getattr(raw_request.state, "request_metadata", None)
if meta is not None and getattr(meta, "request_id", None):
return meta.request_id
return raw_request.headers.get("X-Request-Id")
def _error_response(
error: ErrorResponse,
raw_request: Request | None,
*,
fallback_status: int = HTTPStatus.BAD_REQUEST,
) -> JSONResponse:
"""Translate vLLM's internal error envelope into Cohere's shape."""
info = error.error
status = info.code or fallback_status
return JSONResponse(
status_code=status,
content=CohereError(
message=sanitize_message(info.message),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
@router.post(
"/cohere/v2/chat",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
},
)
@with_cancellation
@load_aware_call
async def chat_v2(request: CohereChatV2Request, raw_request: Request):
handler = _serving(raw_request)
if handler is None:
return JSONResponse(
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
content=CohereError(
message="The model does not support the Cohere v2 chat API.",
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
try:
result = await handler.create_chat_v2(request, raw_request)
except Exception as e:
logger.exception("Error in /cohere/v2/chat: %s", e)
return _error_response(create_error_response(e), raw_request)
match result:
case ErrorResponse():
return _error_response(result, raw_request)
case CohereChatV2Response():
return JSONResponse(content=result.model_dump(exclude_none=True))
case _:
return StreamingResponse(content=result, media_type="text/event-stream")
@render_router.post(
"/cohere/v2/chat/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
},
)
@with_cancellation
async def chat_v2_render(request: CohereChatV2Request, raw_request: Request):
"""Tokenize a Cohere v2 chat request without running generation.
The Cohere counterpart to ``POST /v1/chat/completions/render``.
The v2 body goes through the same conversion
``/cohere/v2/chat`` uses and is then handed to the shared
:class:`ServingRender`, so the returned ``GenerateRequest``
carries the prompt tokens and sampling params the chat endpoint
would have sent to the engine.
"""
handler = _serving(raw_request)
render_handler = _serving_render(raw_request)
if handler is None or render_handler is None:
return JSONResponse(
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
content=CohereError(
message=(
"The model does not support the Cohere v2 chat render API."
),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
try:
chat_request = handler.to_chat_completion_request(request)
result = await render_handler.render_chat_request(chat_request)
except Exception as e:
logger.exception("Error in /cohere/v2/chat/render: %s", e)
return _error_response(create_error_response(e), raw_request)
if isinstance(result, ErrorResponse):
return _error_response(result, raw_request)
if (kwargs := result.reasoning_parser_kwargs) is not None:
kwargs.chat_template_kwargs = handler._engine_chat_template_kwargs(
kwargs.chat_template_kwargs
)
return JSONResponse(content=result.model_dump())
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
The endpoint handler above already returns :class:`CohereError` for
errors it owns, but globally-registered exception handlers (e.g.
:func:`validation_exception_handler` for pydantic body errors,
:func:`http_exception_handler`, engine error handlers) fire *before*
the handler runs and produce vLLM's internal
``ErrorResponse`` shape (``{"error": {"message": ...}}``). That
shape doesn't match the ``CohereError`` schema advertised on the
route's OpenAPI ``responses``, so clients (and schema-conformance
tests like ``test_openai_schema.py``) would see a mismatch on
those paths. This middleware normalises any error body on
``/cohere/*`` responses to :class:`CohereError`.
"""
async def dispatch(self, request: Request, call_next):
response = await call_next(request)
if not request.url.path.startswith(_COHERE_PATH_PREFIX):
return response
if response.status_code < 400:
return response
content_type = response.headers.get("content-type", "")
if not content_type.startswith("application/json"):
return response
body = b"".join([chunk async for chunk in response.body_iterator])
translated = _translate_vllm_error_body(body, request)
if translated is not None:
return translated
passthrough_headers = {
k: v
for k, v in response.headers.items()
if k.lower() != "content-length"
}
return Response(
content=body,
status_code=response.status_code,
headers=passthrough_headers,
media_type=content_type,
)
def _translate_vllm_error_body(raw: bytes, request: Request) -> JSONResponse | None:
"""Translate a vLLM ``ErrorResponse`` body to a ``CohereError`` body.
Returns ``None`` if ``raw`` does not match the vLLM error envelope
(which signals the middleware to pass the body through unchanged).
"""
try:
data = json.loads(raw)
except (json.JSONDecodeError, TypeError, ValueError):
return None
if not (
isinstance(data, dict)
and isinstance(data.get("error"), dict)
and "message" in data["error"]
):
return None
try:
err = ErrorResponse.model_validate(data)
except Exception: # noqa: BLE001 - malformed envelope; pass through
return None
return _error_response(err, request)
def attach_router(app: FastAPI) -> None:
"""Register the enabled ``/cohere/v2/chat`` routes on ``app``.
No-op when either:
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
Cohere v2 endpoints are opt-in because they carry Cohere-specific
request/response semantics (grounding citations, tool_plan,
PLAN/THINKING_CONTENT blocks) that are only meaningful when
serving a Cohere Command-family model.
* the optional ``cohere`` SDK isn't installed (the v2 protocol
models live there)
The two skip paths log at different levels: an operator who set
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
a WARNING (they explicitly asked for the endpoints and they're
silently absent), whereas the default-off skip logs at debug.
"""
enabled = envs.VLLM_ENABLE_COHERE_API
if not enabled:
logger.debug(
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints "
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them."
)
return
if not _SDK_AVAILABLE:
logger.warning(
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
"installed; /cohere/v2/chat will not be exposed. Install "
"with `pip install cohere` to enable the endpoints."
)
return
app.include_router(router)
args = getattr(app.state, "args", None)
if getattr(args, "enable_scale_out", False) or getattr(args, "tokens_only", False):
app.include_router(render_router)
app.add_middleware(CohereErrorEnvelopeMiddleware)
@@ -1,144 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM chat-protocol extensions for grounded (citation-carrying) models.
This module keeps the OpenAI chat completion protocol classes
(:class:`ChatMessage` / :class:`DeltaMessage`) free of Cohere-specific fields.
It provides two things:
* :class:`Citation` / :class:`CitationSource`: the vLLM-internal citation
representation produced by the Cohere parser
(:mod:`vllm.parser.cohere_command`) and consumed by
:mod:`vllm.entrypoints.cohere.serving`. This is *not* the on-wire Cohere
SDK shape -- that conversion happens in the serving layer.
* :class:`CohereChatMessage` / :class:`CohereDeltaMessage`: :class:`ChatMessage`
/ :class:`DeltaMessage` subclasses that add a ``citations`` field. They
are only instantiated by the citation-aware serving handler
(:class:`vllm.entrypoints.cohere.serving.CohereServingChatV2`) and by the
Cohere reasoning parser's streaming path. Non-Cohere code paths continue
to construct plain :class:`ChatMessage` / :class:`DeltaMessage`.
The module intentionally does *not* import the ``cohere`` Python SDK so the
reasoning parser (which is Cohere-model-specific but ships without the SDK
dependency) can import from here freely.
"""
from __future__ import annotations
from typing import Any, Literal
from pydantic import Field, model_serializer
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
class CitationSource(OpenAIBaseModel):
"""Source attribution for a :class:`Citation`.
Mirrors the shape used by Cohere's Chat v2 API. ``type`` is the source
discriminator (``document`` or ``tool``); ``id`` is the citing
document/tool-output identifier; ``document`` and ``tool_output`` carry
the original payload that produced the citation.
Sources are fully resolved by the parser at emit time
(see :func:`_melody_sources_to_vllm` in
:mod:`vllm.parser.cohere_command`) using the
``POSITION_TO_SOURCE_KEY`` map forwarded through
``chat_template_kwargs`` by
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._apply_cohere_template_kwargs`.
That means every instance that reaches the serving / wire layer
already has ``type`` / ``id`` populated; the serving layer only has
to coerce to :class:`cohere.types.Citation` and rewrite
``THINKING_CONTENT`` -> ``PLAN`` for non-reasoning models.
"""
type: Literal["document", "tool"] | None = None
id: str | None = None
document: dict[str, Any] | None = None
tool_output: dict[str, Any] | None = None
class Citation(OpenAIBaseModel):
"""A citation grounding a span of generated text in source material.
vLLM-internal representation used by the Cohere reasoning parser and
the Cohere v2 serving layer. This is not the on-wire Cohere SDK shape:
conversion to the SDK's ``cohere.types.Citation`` happens in
:mod:`vllm.entrypoints.cohere.serving`.
"""
start: int | None = None
"""Start character offset in the surrounding text content."""
end: int | None = None
"""End character offset (exclusive) in the surrounding text content."""
text: str | None = None
"""The cited text snippet."""
sources: list[CitationSource] = Field(default_factory=list)
"""Source documents / tool outputs that ground this citation."""
content_index: int | None = None
"""Index of the content block this citation refers to (when the message
has multiple content blocks)."""
type: Literal["TEXT_CONTENT", "THINKING_CONTENT", "PLAN"] | None = None
"""Which kind of content block this citation grounds: the user-visible
text (``TEXT_CONTENT``), a thinking block (``THINKING_CONTENT``), or a
tool-plan block (``PLAN``). ``None`` means unspecified."""
class CohereChatMessage(ChatMessage):
""":class:`ChatMessage` extension carrying grounding citations.
Only instantiated by :class:`CohereServingChatV2` (and any other
:class:`OpenAIServingChat` subclass that opts in by overriding
``_create_chat_message``). Regular OpenAI-compatible handlers keep
emitting plain :class:`ChatMessage` so their response schema is
unchanged.
The response envelope declares ``message: SerializeAsAny[ChatMessage]``,
so pydantic serializes this subclass with its own schema (including
``citations``) when it flows through
:class:`ChatCompletionResponseChoice`.
"""
citations: list[Citation] | None = None
@model_serializer(mode="wrap")
def _serialize(self, handler):
# ``mode="wrap"`` fully overrides (rather than chains) the
# parent's ``@model_serializer``, so we explicitly delegate via
# ``super()._serialize(handler)`` to preserve the parent's
# cleanup (e.g. stripping empty ``tool_calls``). Then we drop an
# unset ``citations`` field so the wire matches the OpenAI-style
# contract that optional vLLM extensions are omitted rather than
# serialized as ``null``.
data = super()._serialize(handler)
if not data.get("citations"):
data.pop("citations", None)
return data
class CohereDeltaMessage(DeltaMessage):
""":class:`DeltaMessage` extension carrying grounding citations for streaming.
Emitted by :class:`vllm.parser.cohere_command.CohereCommandParser`
on delta events whose payload includes citations. Non-Cohere parsers
return plain :class:`DeltaMessage`, so their streamed shape is
unchanged.
The response envelope declares
``delta: SerializeAsAny[DeltaMessage]``, so pydantic serializes this
subclass with its own schema when it flows through
:class:`ChatCompletionResponseStreamChoice`.
"""
citations: list[Citation] | None = None
@model_serializer(mode="wrap")
def _serialize(self, handler):
# See ``CohereChatMessage._serialize`` for why we delegate via
# ``super()`` instead of calling ``handler(self)`` directly.
data = super()._serialize(handler)
if not data.get("citations"):
data.pop("citations", None)
return data
-389
View File
@@ -1,389 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cohere Chat v2 API protocol.
The bulk of the wire types come straight from the official ``cohere``
Python SDK so we stay in sync with the upstream specification and
avoid re-declaring the schema. We only own three things locally:
1. The top-level request body model (the SDK doesn't ship one — its
``ClientV2.chat`` takes the body as kwargs), with vLLM-specific
extensions (``kv_transfer_params`` / ``chat_template_kwargs``).
2. The non-streaming response envelope (the SDK exposes the message
shape via :class:`AssistantMessageResponse` but no full response
wrapper).
3. The streaming discriminated union (the SDK exports each event type
individually but not as a combined ``Annotated[Union[...],
discriminator]``).
Importing this module pulls in the ``cohere`` package. The router that
mounts ``POST /cohere/v2/chat`` guards on that import succeeding so vLLM still
boots without the SDK installed.
See https://docs.cohere.com/reference/chat for the upstream spec.
"""
from __future__ import annotations
from typing import Annotated, Any, Literal
from cohere import types as _sdk
from cohere.types import (
AssistantChatMessageV2,
AssistantMessageResponse,
ChatMessageV2,
ChatRequestSafetyMode,
Citation,
CitationOptions,
Document,
ResponseFormatV2,
SystemChatMessageV2,
Thinking,
ToolCallV2,
ToolChatMessageV2,
ToolV2,
UserChatMessageV2,
)
from pydantic import BaseModel, Field, field_validator, model_validator
import vllm.envs as envs
from vllm.exceptions import VLLMValidationError
# Re-export the SDK wire-format types alongside our local extensions so
# ``vllm.entrypoints.cohere.serving`` and friends can import everything
# they need from this module.
__all__ = [
"AssistantChatMessageV2",
"AssistantMessageResponse",
"ChatMessageV2",
"Citation",
"CitationEndEvent",
"CitationOptions",
"CitationStartEvent",
"CohereChatV2Request",
"CohereChatV2Response",
"CohereError",
"CohereFinishReason",
"CohereLogprobItem",
"CohereUsage",
"CohereUsageBilledUnits",
"CohereUsageTokens",
"ContentDeltaEvent",
"ContentEndEvent",
"ContentStartEvent",
"Document",
"MessageEndEvent",
"MessageStartEvent",
"ResponseFormatV2",
"SystemChatMessageV2",
"Thinking",
"ToolCallDeltaEvent",
"ToolCallEndEvent",
"ToolCallStartEvent",
"ToolCallV2",
"ToolChatMessageV2",
"ToolPlanDeltaEvent",
"ToolV2",
"UserChatMessageV2",
]
# ---------------------------------------------------------------------------
# Errors
# ---------------------------------------------------------------------------
class CohereError(BaseModel):
"""Top-level error body returned by ``/cohere/v2/chat`` error responses.
Cohere's documented error schemas are uniform: ``{message, id}``.
"""
message: str
id: str | None = None
# ---------------------------------------------------------------------------
# Tool choice / finish reasons
# ---------------------------------------------------------------------------
#
# These literals aren't first-class enums in the SDK but are documented at
# https://docs.cohere.com/reference/chat. We declare them here so the
# request/response models can validate them.
CohereToolChoice = Literal["REQUIRED", "NONE"]
CohereFinishReason = Literal[
"COMPLETE",
"STOP_SEQUENCE",
"MAX_TOKENS",
"TOOL_CALL",
"ERROR",
"TIMEOUT",
]
# ---------------------------------------------------------------------------
# Request
# ---------------------------------------------------------------------------
class CohereChatV2Request(BaseModel):
"""Cohere Chat v2 request body.
Mirrors the schema documented at https://docs.cohere.com/reference/chat.
All structured fields delegate to the official SDK types so the body
schema stays in sync with the upstream spec.
"""
model: str
messages: list[ChatMessageV2]
stream: bool | None = False
# Tooling
tools: list[ToolV2] | None = None
strict_tools: bool | None = None
tool_choice: CohereToolChoice | None = None
# Grounding
documents: list[str | Document] | None = None
citation_options: CitationOptions | None = None
# Output
response_format: ResponseFormatV2 | None = None
safety_mode: ChatRequestSafetyMode | None = None
max_tokens: int | None = None
stop_sequences: (
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
) = None
# Sampling
temperature: float | None = None
seed: int | None = None
frequency_penalty: float | None = None
presence_penalty: float | None = None
k: int | None = None
p: float | None = None
logprobs: bool | None = None
# Reasoning
thinking: Thinking | None = None
# Scheduling
priority: int | None = Field(default=None, ge=-(2**63), le=2**63 - 1)
# vLLM-specific extensions (not in Cohere spec). These mirror what the
# Anthropic and OpenAI surfaces already expose so V2 callers can reach
# the same engine knobs when needed.
watermarking: bool | None = Field(
default=None,
description=(
"Whether to apply the engine's configured watermark to this request."
),
)
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model")
@classmethod
def _validate_model(cls, v: str) -> str:
if not v:
raise VLLMValidationError("model is required", parameter="model")
return v
@field_validator("max_tokens")
@classmethod
def _validate_max_tokens(cls, v: int | None) -> int | None:
if v is not None and v < 0:
raise VLLMValidationError(
"max_tokens must be non-negative",
parameter="max_tokens",
value=v,
)
return v
@model_validator(mode="after")
def _validate_tool_choice(self) -> CohereChatV2Request:
if self.tool_choice == "REQUIRED" and not self.tools:
raise VLLMValidationError(
'tool_choice="REQUIRED" requires at least one tool in tools.',
parameter="tool_choice",
value=self.tool_choice,
)
return self
@field_validator("messages", mode="before")
@classmethod
def _normalize_message_roles(cls, v: Any) -> Any:
"""Rewrite OpenAI-style ``developer`` roles to ``system``.
Cohere's v2 ``ChatMessageV2`` discriminated union only admits
the four literal roles ``user`` / ``assistant`` / ``system`` /
``tool``. OpenAI's ``developer`` role is documented as
high-priority system instructions, so we alias it onto
``system`` *before* the SDK discriminator runs (otherwise it
rejects the message with a ``literal_error`` against each union
member). Mirrors ``_role_to_melody`` in the renderer so the
same alias is honoured no matter which surface the message
arrives through.
``mode="before"`` is required so the rewrite happens before the
``list[ChatMessageV2]`` coercion runs the SDK's discriminated
union; a default-mode validator would never see ``developer``
because validation would have already failed. On any
structural malformation (non-iterable input, items without a
dict-shaped ``role`` field, etc.) we hand ``v`` back unchanged
and let Pydantic's normal coercion surface a precise error.
"""
try:
return [
{**msg, "role": "system"}
if msg.get("role", "").lower() == "developer"
else msg
for msg in v
]
except (AttributeError, TypeError):
return v
@field_validator("messages")
@classmethod
def _validate_messages(cls, v: list[ChatMessageV2]) -> list[ChatMessageV2]:
if not v:
raise VLLMValidationError(
"messages must contain at least one message", parameter="messages"
)
return v
# ---------------------------------------------------------------------------
# Usage / Logprobs
# ---------------------------------------------------------------------------
#
# The Cohere SDK only exposes a v1 ``ApiMetaBilledUnits``. The v2 usage
# envelope is documented separately in the OpenAPI spec and we declare it
# here.
class CohereUsageBilledUnits(BaseModel):
input_tokens: float | None = None
output_tokens: float | None = None
search_units: float | None = None
classifications: float | None = None
class CohereUsageTokens(BaseModel):
input_tokens: float | None = None
output_tokens: float | None = None
class CohereUsage(BaseModel):
billed_units: CohereUsageBilledUnits | None = None
tokens: CohereUsageTokens | None = None
cached_tokens: float | None = None
class CohereLogprobItem(BaseModel):
text: str | None = None
token_ids: list[int]
logprobs: list[float] | None = None
# ---------------------------------------------------------------------------
# Non-streaming response
# ---------------------------------------------------------------------------
class CohereChatV2Response(BaseModel):
"""Cohere Chat v2 non-streaming response body.
Wraps the SDK :class:`AssistantMessageResponse` (the message shape) in
the documented v2 response envelope (``id``, ``finish_reason``,
``usage``, ``logprobs``). The single constructor in
:class:`CohereServingChatV2._chat_completion_to_v2` is responsible for
supplying a non-empty ``id`` (falling back to a synthesized one if
the upstream response is missing it) to this model.
"""
id: str
finish_reason: CohereFinishReason
message: AssistantMessageResponse
usage: CohereUsage | None = None
logprobs: list[CohereLogprobItem] | None = None
# vLLM-specific extension.
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
# ---------------------------------------------------------------------------
# Streaming events
# ---------------------------------------------------------------------------
#
# Cohere V2 streams a sequence of typed JSON events delivered as Server-
# Sent Events; each event's ``type`` field carries the discriminator. The
# SDK exposes a Pydantic model per event but none of them declare ``type``
# as a field (the SDK relies on its own deserializer for discrimination),
# so a naive ``model_dump_json()`` would silently drop the discriminator
# and break clients that demux on ``type``.
#
# We therefore subclass each SDK event and bake the wire-format ``type``
# string in as a ``Literal`` field with a default. ``model_dump()`` now
# emits ``type`` for free, and surfaces can simply construct the event
# class and serialize it -- no manual ``type`` parameter needed.
#
# See https://docs.cohere.com/v2/docs/streaming and the OpenAPI
# ``StreamedChatResponseV2`` schema for the wire-format reference.
class MessageStartEvent(_sdk.ChatMessageStartEvent):
type: Literal["message-start"] = "message-start"
class ContentStartEvent(_sdk.ChatContentStartEvent):
type: Literal["content-start"] = "content-start"
class ContentDeltaEvent(_sdk.ChatContentDeltaEvent):
type: Literal["content-delta"] = "content-delta"
class ContentEndEvent(_sdk.ChatContentEndEvent):
type: Literal["content-end"] = "content-end"
class ToolPlanDeltaEvent(_sdk.ChatToolPlanDeltaEvent):
type: Literal["tool-plan-delta"] = "tool-plan-delta"
class ToolCallStartEvent(_sdk.ChatToolCallStartEvent):
type: Literal["tool-call-start"] = "tool-call-start"
class ToolCallDeltaEvent(_sdk.ChatToolCallDeltaEvent):
type: Literal["tool-call-delta"] = "tool-call-delta"
class ToolCallEndEvent(_sdk.ChatToolCallEndEvent):
type: Literal["tool-call-end"] = "tool-call-end"
class CitationStartEvent(_sdk.CitationStartEvent):
type: Literal["citation-start"] = "citation-start"
class CitationEndEvent(_sdk.CitationEndEvent):
type: Literal["citation-end"] = "citation-end"
class MessageEndEvent(_sdk.ChatMessageEndEvent):
type: Literal["message-end"] = "message-end"
File diff suppressed because it is too large Load Diff
+1 -5
View File
@@ -31,7 +31,6 @@ from vllm.renderers.inputs.preprocess import (
)
from vllm.sampling_params import RequestOutputKind
from vllm.utils.counter import Counter
from vllm.utils.mistral import is_mistral_tokenizer
from vllm.utils.tqdm_utils import maybe_tqdm
from vllm.v1.engine.llm_engine import LLMEngine
@@ -191,10 +190,7 @@ class OfflineInferenceMixin:
add_generation_prompt=add_generation_prompt,
continue_final_message=continue_final_message,
tools=tools,
tokenize=(
is_mistral_tokenizer(renderer.tokenizer)
or self.model_config.enable_prompt_embeds
),
tokenize=self.model_config.enable_prompt_embeds,
),
),
mm_processor_kwargs=mm_processor_kwargs,
+1 -79
View File
@@ -4,8 +4,6 @@ from typing import TYPE_CHECKING, Any
from fastapi import FastAPI
import vllm.envs as envs
if TYPE_CHECKING:
from argparse import Namespace
@@ -37,17 +35,6 @@ def register_generate_api_routers(app: FastAPI):
register_completion_api_router(app)
from vllm.entrypoints.anthropic.api_router import (
attach_router as register_anthropic_api_router,
)
register_anthropic_api_router(app)
from vllm.entrypoints.cohere.api_router import (
attach_router as register_cohere_api_router,
)
register_cohere_api_router(app)
from .generative_scoring.api_router import register_generative_scoring_api_router
@@ -62,24 +49,6 @@ async def init_generate_state(
supported_tasks: tuple["SupportedTask", ...],
default_chat_template_kwargs: dict[str, Any],
):
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.renderers.chat_utils import load_chat_template
# The Cohere serving handler depends on the optional `cohere` SDK for
# its wire-format protocol models, and is additionally gated on the
# `VLLM_ENABLE_COHERE_API` env flag (see
# `vllm.entrypoints.cohere.api_router.attach_router`). Skip the import
# entirely when the endpoint isn't going to be exposed, both because
# the SDK may not be installed and because the serving object holds
# nontrivial state (chat handler, warmup) that would otherwise be
# unused.
if envs.VLLM_ENABLE_COHERE_API:
try:
from vllm.entrypoints.cohere.serving import CohereServingChatV2
except ImportError:
CohereServingChatV2 = None # type: ignore[assignment,misc]
else:
CohereServingChatV2 = None # type: ignore[assignment,misc]
from vllm.entrypoints.openai.chat_completion.batch_serving import (
OpenAIServingChatBatch,
)
@@ -87,6 +56,7 @@ async def init_generate_state(
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
from vllm.renderers.chat_utils import load_chat_template
# Applied before any serving class is constructed so that each one picks
# up the chosen mode on its first cache miss.
@@ -168,54 +138,6 @@ async def init_generate_state(
if "generate" in supported_tasks
else None
)
state.anthropic_serving_messages = (
AnthropicServingMessages(
engine_client,
state.openai_serving_models,
args.response_role,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
enable_log_deltas=args.enable_log_deltas,
default_chat_template_kwargs=default_chat_template_kwargs,
disabled_thinking_effort=args.anthropic_disabled_thinking_effort,
)
if "generate" in supported_tasks
else None
)
state.cohere_serving_chat_v2 = (
CohereServingChatV2(
engine_client,
state.openai_serving_models,
args.response_role,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
enable_log_deltas=args.enable_log_deltas,
default_chat_template_kwargs=default_chat_template_kwargs,
is_reasoning_model=args.cohere_is_reasoning_model,
)
if CohereServingChatV2 is not None and "generate" in supported_tasks
else None
)
from .generative_scoring.serving import ServingGenerativeScoring
-42
View File
@@ -1,42 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.config import ModelConfig
from vllm.tasks import SupportedTask
if TYPE_CHECKING:
from vllm.entrypoints.serve.sagemaker.api_router import (
EndpointFn,
GetHandlerFn,
RequestType,
)
def get_generate_invocation_types(
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
# NOTE: Items defined earlier take higher priority
invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
if "generate" in supported_tasks:
from vllm.entrypoints.openai.chat_completion.api_router import (
chat,
create_chat_completion,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.api_router import (
completion,
create_completion,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
invocation_types += [
(ChatCompletionRequest, (chat, create_chat_completion)),
(CompletionRequest, (completion, create_completion)),
]
return invocation_types
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,86 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from http import HTTPStatus
from typing import Any
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.decisions.state import init_decisions_state
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.renderers.chat_utils import load_chat_template
from vllm.tasks import SupportedTask
from .protocol import StructuredDecisionRequest
from .serving import ServingStructuredDecisions
router = APIRouter()
def structured_decisions(request: Request) -> ServingStructuredDecisions | None:
return getattr(request.app.state, "serving_structured_decisions", None)
@router.post(
"/v1/systemone",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_structured_decision(
request: StructuredDecisionRequest, raw_request: Request
):
handler = structured_decisions(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support the structured decision API"
)
result = await handler.create_decision(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def register_structured_decisions_api_router(app: FastAPI):
app.include_router(router)
def init_structured_decisions_state(
engine_client: EngineClient,
state: State,
args: Namespace,
request_logger: RequestLogger | None,
supported_tasks: tuple[SupportedTask, ...],
default_chat_template_kwargs: dict[str, Any],
):
strategy = init_decisions_state(
engine_client,
state,
args,
request_logger,
load_chat_template(args.chat_template),
default_chat_template_kwargs,
supported_tasks,
)
state.serving_structured_decisions = (
ServingStructuredDecisions(
state.openai_serving_models, strategy, request_logger=request_logger
)
if strategy is not None
else None
)
@@ -1,83 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Request and response models of the structured decisions API (/v1/systemone)."""
import time
from typing import Any, Literal
from pydantic import Field, model_validator
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
from vllm.utils import random_uuid
class QuestionSpec(OpenAIBaseModel):
type: str
instructions: Any = ""
criteria: Any = None
class StructuredDecisionRequest(OpenAIBaseModel):
model: str | None = None
state: Any = Field(..., description="What the questions are about: text or JSON.")
questions: dict[str, QuestionSpec] = Field(
..., description="Question id to question, asked in this order."
)
instructions: str | None = Field(
default=None, description="Context placed ahead of the questions."
)
images: list[str] | None = Field(
default=None, description="Image URLs or data URLs, for models that see."
)
audio: list[str] | None = Field(
default=None, description="Audio URLs or data URLs, for models that hear."
)
videos: list[str] | None = Field(
default=None, description="Video URLs or data URLs, for models that see."
)
chat_template_kwargs: dict[str, Any] | None = None
cache_salt: str | None = Field(
default=None,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
request_id: str = Field(default_factory=random_uuid)
@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
if isinstance(data, dict):
validate_cache_salt(data.get("cache_salt"))
return data
class DecisionUsage(OpenAIBaseModel):
input_tokens: int
output_tokens: int
class QuestionDiagnostics(OpenAIBaseModel):
label_mass: float = Field(
description="Probability the model put on the labels, over the full "
"vocabulary. A low value means most of the probability went to tokens "
"that are not labels."
)
argmax_is_label: bool
class StructuredDecisionResponse(OpenAIBaseModel):
id: str
object: Literal["structured_decision"] = "structured_decision"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
answers: dict[str, dict[str, Any]]
usage: DecisionUsage
diagnostics: dict[str, QuestionDiagnostics]
@@ -1,123 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from typing import Any
from fastapi import Request
from vllm.entrypoints.openai.decisions.question_types import (
Question,
StructuredDecisionError,
build_question,
)
from vllm.entrypoints.openai.decisions.serving import BaseServingDecisions
from vllm.entrypoints.openai.decisions.strategies import DecisionLimits
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from .protocol import (
DecisionUsage,
QuestionDiagnostics,
StructuredDecisionRequest,
StructuredDecisionResponse,
)
def parse_questions(
request: StructuredDecisionRequest,
limits: DecisionLimits,
) -> list[Question]:
if not request.questions:
raise StructuredDecisionError("questions: needs at least one question")
if len(request.questions) > limits.max_questions:
raise StructuredDecisionError(
f"questions: at most {limits.max_questions} for this model"
)
questions = []
for qid, spec in request.questions.items():
if spec.model_extra:
raise StructuredDecisionError(
f"question {qid!r}: unknown field(s) {sorted(spec.model_extra)}"
)
questions.append(
build_question(
qid,
spec.type,
spec.instructions,
spec.criteria,
limits.max_options,
labeled=limits.labeled,
)
)
return questions
class ServingStructuredDecisions(BaseServingDecisions):
def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]:
urls = {
"image": request.images,
"audio": request.audio,
"video": request.videos,
}
media = {modality: items for modality, items in urls.items() if items}
if unsupported := media.keys() - self.strategy.modalities:
names = ", ".join(sorted(unsupported))
raise StructuredDecisionError(f"this model does not take {names} inputs")
return media
async def create_decision(
self,
request: StructuredDecisionRequest,
raw_request: Request | None = None,
) -> StructuredDecisionResponse | ErrorResponse:
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
return error
engine_client = self.strategy.context.engine_client
if engine_client.errored:
raise engine_client.dead_error
base_id = self._base_request_id(raw_request, default=request.request_id)
request_id = f"decision-{base_id}"
try:
questions = parse_questions(request, self.limits)
media = self._media(request)
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
engine_client.check_admission(self.strategy.num_requests(questions))
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
reads = await self.strategy.read(
questions,
request.instructions,
self.strategy.render_state(request.state),
request_id=request_id,
chat_template_kwargs=request.chat_template_kwargs,
lora_request=lora_request,
priority=request.priority,
cache_salt=request.cache_salt,
trace_headers=trace_headers,
media=media,
)
except StructuredDecisionError as e:
return self.create_error_response(e)
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
answers: dict[str, dict[str, Any]] = {}
diagnostics: dict[str, QuestionDiagnostics] = {}
for q, read in zip(questions, reads):
answers[q.id] = q.type.answer(q, read.probs, read.label_mass)
diagnostics[q.id] = QuestionDiagnostics(
label_mass=read.label_mass, argmax_is_label=read.argmax_is_label
)
return StructuredDecisionResponse(
id=request_id,
model=self.models.model_name(lora_request),
answers=answers,
usage=DecisionUsage(
input_tokens=sum(r.input_tokens for r in reads),
output_tokens=sum(r.output_tokens for r in reads),
),
diagnostics=diagnostics,
)
@@ -7,7 +7,6 @@ from typing import cast
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
@@ -16,7 +15,6 @@ from vllm.entrypoints.serve.utils.api_utils import process_lora_modules
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
from vllm.renderers.chat_utils import load_chat_template
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
@@ -65,7 +63,7 @@ async def init_app_state(
state.vllm_config = vllm_config
state.args = args
resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
state.tool_server = (
await init_tool_server(args) if "generate" in supported_tasks else None
)
@@ -103,22 +101,6 @@ async def init_app_state(
)
state.online_renderer.warmup()
state.online_derenderer = OnlineDerenderer(
model_config=engine_client.model_config,
renderer=engine_client.renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.serving_tokenization = ServingTokenization(
state.openai_serving_models,
state.online_renderer,
@@ -152,19 +134,6 @@ async def init_app_state(
engine_client, state, args, request_logger, supported_tasks
)
from vllm.entrypoints.generate.structured_decisions.api_router import (
init_structured_decisions_state,
)
init_structured_decisions_state(
engine_client,
state,
args,
request_logger,
supported_tasks,
default_chat_template_kwargs,
)
await init_endpoint_plugins_state(engine_client, state, args)
state.enable_server_load_tracking = args.enable_server_load_tracking
@@ -25,33 +25,11 @@ def register_api_routers(
register_models_api_router(app)
from vllm.entrypoints.generate.structured_decisions.api_router import (
register_structured_decisions_api_router,
)
register_structured_decisions_api_router(app)
from vllm.entrypoints.openai.decisions.api_router import (
register_decisions_api_router,
)
register_decisions_api_router(app)
from vllm.entrypoints.serve.sagemaker.api_router import (
attach_router as register_sagemaker_api_router,
)
register_sagemaker_api_router(app, supported_tasks, model_config)
if envs.VLLM_SERVER_DEV_MODE:
from vllm.entrypoints.serve import register_vllm_dev_api_routers
register_vllm_dev_api_routers(app)
from vllm.entrypoints.rl.factories import register_rl_api_routers
register_rl_api_routers(app)
if "generate" in supported_tasks:
from vllm.entrypoints.generate.api_router import (
register_generate_api_routers,
@@ -59,13 +37,6 @@ def register_api_routers(
register_generate_api_routers(app)
from vllm.entrypoints.serve.elastic_ep.api_router import (
attach_router as elastic_ep_attach_router,
)
elastic_ep_attach_router(app)
if "generate" in supported_tasks or "render" in supported_tasks:
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
register_scale_out_api_routers(app, supported_tasks)
+1 -8
View File
@@ -8,7 +8,6 @@ from fastapi import FastAPI
from vllm.config import ModelConfig
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
@@ -35,22 +34,16 @@ def build_app(
app = FastAPI(
openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
)
elif args.enable_offline_docs:
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
else:
app = FastAPI(lifespan=lifespan)
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
app.state.args = args
app.root_path = args.root_path
register_api_routers(args, app, supported_tasks, model_config)
# Endpoint plugin hooks run after core routers so plugins can inspect them.
# This runs even for the CPU only render server. A plugin eligible for
# the `render` task still gets its routes registered. It receives
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
attach_endpoint_plugins(app, supported_tasks)
init_exception_handler(app)
init_entrypoints_middleware(args, app, supported_tasks)
app = sagemaker_standards_bootstrap(app)
return app
+6 -54
View File
@@ -14,9 +14,6 @@ from typing import Any, Literal
import vllm.envs as envs
from vllm.config import config
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
from vllm.entrypoints.anthropic.protocol import (
AnthropicDisabledThinkingEffortOption,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.logger import init_logger
from vllm.renderers.chat_utils import (
@@ -172,32 +169,6 @@ class BaseFrontendArgs:
"""If set to False, output deltas will not be logged. Relevant only if
--enable-log-outputs is set.
"""
cohere_is_reasoning_model: bool = True
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
reasoning Command-family model. When True (default), the assistant's
chain-of-thought is surfaced as a ``thinking`` content block (or
``content-*`` events on the stream). When False, reasoning is
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
events) whenever the model emits tool calls, matching older non-
reasoning Command models. Has no effect on the non-Cohere
endpoints."""
cohere_format: str = "cmd4"
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
format to render: ``cmd4`` (current Command A+ models; default) or
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
wrong format silently produces a prompt the model wasn't trained
on, which most commonly manifests as the model emitting text but no
citations / tool calls / thinking blocks. Equivalent to passing
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
explicit request-level ``chat_template_kwargs.cohere_format`` takes
priority."""
anthropic_disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto"
"""Anthropic ``/v1/messages`` only. The ``reasoning_effort`` used for
requests with ``thinking: {"type": "disabled"}``. ``none`` turns thinking
off for models that support it; ``low`` suits models that always think
(e.g. GLM-5.3) or reject ``none`` (e.g. gpt-oss). ``auto`` (default) uses
``low`` when the renderer rejects ``none`` or renders it the same as a
thinking effort, and ``none`` otherwise."""
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
"""If set to True, log the stack trace of error responses"""
tokens_only: bool = False
@@ -207,10 +178,10 @@ class BaseFrontendArgs:
"""
enable_scale_out: bool = False
"""
If set to True, register the scale-out endpoints (`/render`, `/derender`,
`/inference/v1/generate` and `/inference/v1/abort_requests`) on `vllm serve`.
Has no effect on `vllm launch render` or `vllm serve --tokens-only` which
always register their required endpoints regardless of this flag.
If set to True, register the scale-out endpoints (`/inference/v1/generate`
and `/inference/v1/abort_requests`) on `vllm serve`. `vllm serve
--tokens-only` always registers the endpoints it needs regardless of this
flag.
"""
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
"""Controls the ``system_fingerprint`` field on responses.
@@ -360,11 +331,6 @@ class FrontendArgs(BaseFrontendArgs):
h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
"""Maximum number of HTTP headers allowed in a request for h11 parser.
Helps mitigate header abuse. Default: 256."""
enable_offline_docs: bool = False
"""
Enable offline FastAPI documentation for air-gapped environments.
Uses vendored static assets bundled with vLLM.
"""
enable_flash_late_interaction: bool = True
"""If set, run pooling score MaxSim on GPU in the API server process.
Can significantly improve late-interaction scoring performance.
@@ -404,16 +370,6 @@ class FrontendArgs(BaseFrontendArgs):
return frontend_kwargs
def resolve_default_chat_template_kwargs(
args: argparse.Namespace,
) -> dict[str, Any]:
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
kwargs.setdefault("cohere_format", args.cohere_format)
return kwargs
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"""Create the CLI argument parser used by the OpenAI API server.
@@ -453,9 +409,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
action="store_true",
default=False,
help="Launch the Python gRPC (SMG VllmEngine) server on --port instead "
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. "
"Cannot be combined with --grpc-port, which enables the Rust "
"frontend's gRPC services.",
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc].",
)
parser = FrontendArgs.add_cli_args(parser)
parser = AsyncEngineArgs.add_cli_args(parser)
@@ -465,9 +419,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
def validate_parsed_serve_args(args: argparse.Namespace):
"""Quick checks for model serve args that raise prior to loading."""
# `vllm launch <component>` builds its parser with make_arg_parser too (see
# LaunchSubcommandBase.add_cli_args), so its args are serve args as well.
if hasattr(args, "subparser") and args.subparser not in ("serve", "launch"):
if hasattr(args, "subparser") and args.subparser != "serve":
return
# Ensure that the chat template is valid; raises if it likely isn't
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from starlette.datastructures import State
from vllm.config import VllmConfig
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.scale_out.factories import init_render_state
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
from vllm.renderers import renderer_from_config
from vllm.renderers.chat_utils import load_chat_template
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
async def init_render_app_state(
vllm_config: VllmConfig,
state: State,
args: Namespace,
) -> None:
"""Initialise FastAPI app state for a CPU-only render server.
Unlike :func:`init_app_state` this function does not require an
:class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
preprocessing pipeline (renderer, input_processor)
directly from the :class:`~vllm.config.VllmConfig`.
"""
served_model_names = args.served_model_name or [args.model]
model_registry = OpenAIModelRegistry(
model_config=vllm_config.model_config,
base_model_paths=[
BaseModelPath(name=name, model_path=args.model)
for name in served_model_names
],
)
if args.enable_log_requests:
request_logger = RequestLogger(max_log_len=args.max_log_len)
else:
request_logger = None
renderer = renderer_from_config(vllm_config)
resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
state.tool_server = await init_tool_server(args)
# The config-resolved reasoning parser carries both the CLI flag (merged
# in the entrypoint via `create_structured_outputs_config`) and any
# model-specific default applied by `verify_and_update_config`
# (e.g. "openai_gptoss" for gpt_oss), matching the main API server.
# Keep `args.reasoning_parser` as the first source so callers that build
# a VllmConfig without that merge still honor an explicit flag.
reasoning_parser = (
args.reasoning_parser or vllm_config.structured_outputs_config.reasoning_parser
)
state.online_renderer = OnlineRenderer(
model_config=vllm_config.model_config,
renderer=renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_renderer.warmup()
state.online_derenderer = OnlineDerenderer(
model_config=vllm_config.model_config,
renderer=renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.openai_serving_models = model_registry
state.serving_tokenization = ServingTokenization(
model_registry,
state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
init_render_state(state, request_logger)
state.vllm_config = vllm_config
# Disable stats logging — there is no engine to poll.
state.log_stats = False
state.engine_client = None
state.args = args
state.enable_server_load_tracking = False
state.server_load_metrics = 0
# No `EngineClient` exists for the render server, so plugins get `None` and
# must handle it themselves (see `EndpointPlugin.init_state`).
await init_endpoint_plugins_state(None, state, args)
-129
View File
@@ -1,129 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import asyncio
import signal
import socket
from argparse import Namespace
from vllm import AsyncEngineArgs, envs
from vllm.config import VllmConfig
from vllm.logger import configure_logging_from_args, init_logger
from ..app import build_app
from ..launcher import serve_http, setup_server
from ..utils.server_utils import get_uvicorn_log_config
from .app_state import init_render_app_state
logger = init_logger("vllm.entrypoints.launchers.render.entry")
async def build_and_serve_renderer(
vllm_config: VllmConfig,
listen_address: str,
sock: socket.socket,
args: Namespace,
**uvicorn_kwargs,
) -> asyncio.Task:
"""Build FastAPI app for a CPU-only render server, initialize state, and
start serving.
Returns the shutdown task for the caller to await.
"""
# Get uvicorn log config (from file or with endpoint filter)
log_config = get_uvicorn_log_config(args)
if log_config is not None:
uvicorn_kwargs["log_config"] = log_config
app = build_app(args, ("render",))
await init_render_app_state(vllm_config, app.state, args)
logger.info("Starting vLLM server on %s", listen_address)
return await serve_http(
app,
sock=sock,
enable_ssl_refresh=args.enable_ssl_refresh,
host=args.host,
port=args.port,
log_level=args.uvicorn_log_level,
# NOTE: When the 'disable_uvicorn_access_log' value is True,
# no access log will be output.
access_log=not args.disable_uvicorn_access_log,
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
ssl_keyfile=args.ssl_keyfile,
ssl_certfile=args.ssl_certfile,
ssl_ca_certs=args.ssl_ca_certs,
ssl_cert_reqs=args.ssl_cert_reqs,
ssl_ciphers=args.ssl_ciphers,
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
h11_max_header_count=args.h11_max_header_count,
**uvicorn_kwargs,
)
async def run_launch_fastapi(args: argparse.Namespace) -> None:
"""Run the online serving layer with FastAPI (no GPU inference)."""
# Interrupt initialization if SIGTERM arrives before uvicorn installs
# its own signal handlers. Once uvicorn is running it replaces this.
def _interrupt_init(*_) -> None:
raise KeyboardInterrupt("terminated")
signal.signal(signal.SIGTERM, _interrupt_init)
# 1. Socket binding
listen_address, sock = setup_server(args, reuse_port=False)
# 2. Build and serve the API server
engine_args = AsyncEngineArgs.from_cli_args(args)
model_config = engine_args.create_model_config()
# Render servers preprocess data only — no inference, no quantized kernels.
# Clear quantization so VllmConfig skips quant dtype/capability validation.
model_config.quantization = None
# Render servers never allocate KV cache; suppress the spurious CPU KV
# cache space warning from CpuPlatform.check_and_update_config.
envs.VLLM_CPU_KVCACHE_SPACE = 0
# Merge frontend parser flags (e.g. --reasoning-parser) into the
# structured outputs config, mirroring `create_engine_config`, which this
# GPU-less entrypoint bypasses. Model-specific defaults are then applied
# by `verify_and_update_config` during VllmConfig construction.
vllm_config = VllmConfig(
model_config=model_config,
logging_config=engine_args.create_logging_config(),
structured_outputs_config=engine_args.create_structured_outputs_config(),
)
shutdown_task = await build_and_serve_renderer(
vllm_config, listen_address, sock, args
)
try:
await shutdown_task
finally:
sock.close()
if __name__ == "__main__":
import uvloop
from vllm.entrypoints.serve.utils.api_utils import cli_env_setup
from vllm.utils.argparse_utils import FlexibleArgumentParser
from ..cli_args import (
make_arg_parser,
validate_parsed_serve_args,
)
cli_env_setup()
parser = FlexibleArgumentParser(
description="Starts a GPU-less rendering server "
"for preprocessing and postprocessing only"
)
parser = make_arg_parser(parser)
args = parser.parse_args()
configure_logging_from_args(args)
validate_parsed_serve_args(args)
uvloop.run(run_launch_fastapi(args))
-672
View File
@@ -1,672 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import contextlib
import json
import sys
import tempfile
from argparse import Namespace
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from io import StringIO
from typing import Any, NoReturn, TypeAlias
from urllib.parse import urlparse
import aiohttp
import pybase64 as base64
import pydantic
import torch
from prometheus_client import start_http_server
from pydantic import TypeAdapter, field_validator
from pydantic_core.core_schema import ValidationInfo
from starlette.datastructures import State
from starlette.responses import JSONResponse
from tqdm import tqdm
from urllib3.util import parse_url
import vllm.envs as envs
from vllm.config import config
from vllm.connections import HTTPResponseSizeExceededError, global_http_connection
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import (
ErrorResponse,
OpenAIBaseModel,
)
from vllm.exceptions import VLLMValidationError
from vllm.logger import configure_logging_from_args, init_logger
from vllm.reasoning import ReasoningParserManager
from vllm.utils import random_uuid
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.mem_constants import MiB_bytes
from vllm.version import __version__ as VLLM_VERSION
from .api_server.entry import init_app_state
from .cli_args import BaseFrontendArgs
logger = init_logger(__name__)
def _get_audio_download_limit_bytes() -> int:
return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
raise VLLMValidationError(
"Maximum file size exceeded",
parameter="audio_filesize_mb",
value=size_bytes / MiB_bytes,
)
BatchRequestInputBody: TypeAlias = ChatCompletionRequest
class BatchRequestInput(OpenAIBaseModel):
"""The per-line object of the batch input file.
NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
"""
# A developer-provided per-request id that will be used to match outputs to
# inputs. Must be unique for each request in a batch.
custom_id: str
# The HTTP method to be used for the request. Currently only POST is
# supported.
method: str
# The OpenAI API relative URL to be used for the request. Currently
# /v1/chat/completions is supported.
url: str
# The parameters of the request.
body: BatchRequestInputBody
@field_validator("body", mode="plain")
@classmethod
def check_type_for_url(cls, value: Any, info: ValidationInfo):
# Use url to disambiguate models
url: str = info.data["url"]
if url == "/v1/chat/completions":
return ChatCompletionRequest.model_validate(value)
return TypeAdapter(BatchRequestInputBody).validate_python(value)
AllResponse: TypeAlias = (
ChatCompletionResponse
)
class BatchResponseData(OpenAIBaseModel):
# HTTP status code of the response.
status_code: int = 200
# An unique identifier for the API request.
request_id: str
# The body of the response.
body: AllResponse | None = None
class BatchRequestOutput(OpenAIBaseModel):
"""The per-line object of the batch output and error files"""
id: str
# A developer-provided per-request id that will be used to match outputs to
# inputs.
custom_id: str
response: BatchResponseData | None
# For requests that failed with a non-HTTP error, this will contain more
# information on the cause of the failure.
error: Any | None
@config
class BatchFrontendArgs(BaseFrontendArgs):
"""Arguments for the batch runner frontend."""
input_file: str | None = None
"""The path or url to a single input file. Currently supports local file
paths, or the http protocol (http or https). If a URL is specified,
the file should be available via HTTP GET."""
output_file: str | None = None
"""The path or url to a single output file. Currently supports
local file paths, or web (http or https) urls. If a URL is specified,
the file should be available via HTTP PUT."""
output_tmp_dir: str | None = None
"""The directory to store the output file before uploading it
to the output URL."""
enable_metrics: bool = False
"""Enable Prometheus metrics"""
host: str | None = None
"""Host name for the Prometheus metrics server
(only needed if enable-metrics is set)."""
port: int = 8000
"""Port number for the Prometheus metrics server
(only needed if enable-metrics is set)."""
url: str = "0.0.0.0"
"""[DEPRECATED] Host name for the Prometheus metrics server
(only needed if enable-metrics is set). Use --host instead."""
@classmethod
def _customize_cli_kwargs(
cls,
frontend_kwargs: dict[str, Any],
) -> dict[str, Any]:
frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
frontend_kwargs["input_file"]["flags"] = ["-i"]
frontend_kwargs["input_file"]["required"] = True
frontend_kwargs["output_file"]["flags"] = ["-o"]
frontend_kwargs["output_file"]["required"] = True
frontend_kwargs["enable_metrics"]["action"] = "store_true"
frontend_kwargs["url"]["deprecated"] = True
return frontend_kwargs
def make_arg_parser(parser: FlexibleArgumentParser):
parser = BatchFrontendArgs.add_cli_args(parser)
parser = AsyncEngineArgs.add_cli_args(parser)
return parser
def parse_args():
parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible batch runner.")
args = make_arg_parser(parser).parse_args()
configure_logging_from_args(args)
# Backward compatibility: If --url is set, use it for host
url_explicit = any(arg == "--url" or arg.startswith("--url=") for arg in sys.argv)
host_explicit = any(
arg == "--host" or arg.startswith("--host=") for arg in sys.argv
)
if url_explicit and hasattr(args, "url") and not host_explicit:
args.host = args.url
logger.warning_once(
"Using --url for metrics is deprecated. Please use --host instead."
)
return args
# explicitly use pure text format, with a newline at the end
# this makes it impossible to see the animation in the progress bar
# but will avoid messing up with ray or multiprocessing, which wraps
# each line of output with some prefix.
_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501
class BatchProgressTracker:
def __init__(self):
self._total = 0
self._pbar: tqdm | None = None
def submitted(self):
self._total += 1
def completed(self):
if self._pbar:
self._pbar.update()
def pbar(self) -> tqdm:
enable_tqdm = (
not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
)
self._pbar = tqdm(
total=self._total,
unit="req",
desc="Running batch",
mininterval=5,
disable=not enable_tqdm,
bar_format=_BAR_FORMAT,
)
return self._pbar
async def read_file(path_or_url: str) -> str:
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
async with aiohttp.ClientSession() as session, session.get(path_or_url) as resp:
resp.raise_for_status()
return await resp.text()
else:
with open(path_or_url, encoding="utf-8") as f:
return f.read()
async def write_local_file(
output_path: str, batch_outputs: list[BatchRequestOutput]
) -> None:
"""Write the responses to a local file.
output_path: The path to write the responses to.
batch_outputs: The list of batch outputs to write.
"""
# We should make this async, but as long as run_batch runs as a
# standalone program, blocking the event loop won't affect performance.
with open(output_path, "w", encoding="utf-8") as f:
for o in batch_outputs:
print(o.model_dump_json(), file=f)
async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> None:
"""Upload a local file to a URL.
output_url: The URL to upload the file to.
data_or_file: Either the data to upload or the path to the file to upload.
from_file: If True, data_or_file is the path to the file to upload.
"""
# Timeout is a common issue when uploading large files.
# We retry max_retries times before giving up.
max_retries = 5
# Number of seconds to wait before retrying.
delay = 5
for attempt in range(1, max_retries + 1):
try:
# We increase the timeout to 1000 seconds to allow
# for large files (default is 300).
async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=1000)
) as session:
if from_file:
with open(data_or_file, "rb") as file:
async with session.put(output_url, data=file) as response:
if not 200 <= response.status < 300:
error_text = await response.text()
raise Exception(
f"Failed to upload file.\n"
f"Status: {response.status}\n"
f"Response: {error_text}"
)
else:
async with session.put(output_url, data=data_or_file) as response:
if not 200 <= response.status < 300:
error_text = await response.text()
raise Exception(
f"Failed to upload data.\n"
f"Status: {response.status}\n"
f"Response: {error_text}"
)
# Upload succeeded; do not retry.
return
except Exception as e:
if attempt < max_retries:
logger.error(
"Failed to upload data (attempt %d). Error message: %s.\nRetrying in %d seconds...", # noqa: E501
attempt,
e,
delay,
)
await asyncio.sleep(delay)
else:
raise Exception(
f"Failed to upload data (attempt {attempt}). Error message: {str(e)}." # noqa: E501
) from e
async def write_file(
path_or_url: str, batch_outputs: list[BatchRequestOutput], output_tmp_dir: str
) -> None:
"""Write batch_outputs to a file or upload to a URL.
path_or_url: The path or URL to write batch_outputs to.
batch_outputs: The list of batch outputs to write.
output_tmp_dir: The directory to store the output file before uploading it
to the output URL.
"""
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
if output_tmp_dir is None:
logger.info("Writing outputs to memory buffer")
output_buffer = StringIO()
for o in batch_outputs:
print(o.model_dump_json(), file=output_buffer)
output_buffer.seek(0)
logger.info("Uploading outputs to %s", path_or_url)
await upload_data(
path_or_url,
output_buffer.read().strip().encode("utf-8"),
from_file=False,
)
else:
# Write responses to a temporary file and then upload it to the URL.
with tempfile.NamedTemporaryFile(
mode="w",
encoding="utf-8",
dir=output_tmp_dir,
prefix="tmp_batch_output_",
suffix=".jsonl",
) as f:
logger.info("Writing outputs to temporary local file %s", f.name)
await write_local_file(f.name, batch_outputs)
logger.info("Uploading outputs to %s", path_or_url)
await upload_data(path_or_url, f.name, from_file=True)
else:
logger.info("Writing outputs to local file %s", path_or_url)
await write_local_file(path_or_url, batch_outputs)
async def download_bytes_from_url(
url: str,
allowed_media_domains: list[str] | None = None,
) -> bytes:
"""Download data from a URL or decode from a data URL.
Args:
url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
allowed_media_domains: If set, only HTTP/HTTPS URLs whose hostname
is in this list are permitted. data: URLs are not subject to
this restriction.
Returns:
Data as bytes
"""
parsed = urlparse(url)
# Handle data URLs (base64 encoded) - not subject to domain restrictions
if parsed.scheme == "data":
# Format: data:...;base64,<base64_data>
if "," in url:
header, data = url.split(",", 1)
if "base64" in header:
decoded_size_bytes = len(data.rstrip("=")) * 3 // 4
if decoded_size_bytes > _get_audio_download_limit_bytes():
_raise_audio_filesize_limit(decoded_size_bytes)
return base64.b64decode(data)
else:
raise VLLMValidationError(
f"Unsupported data URL encoding: {header}",
parameter="url",
)
else:
raise VLLMValidationError(
f"Invalid data URL format: {url}",
parameter="url",
)
# Handle HTTP/HTTPS URLs
elif parsed.scheme in ("http", "https"):
if allowed_media_domains is not None:
url_spec = parse_url(url)
if url_spec.hostname not in allowed_media_domains:
raise VLLMValidationError(
f"The URL must be from one of the allowed domains: "
f"{allowed_media_domains}. Input URL domain: "
f"{url_spec.hostname}",
parameter="url",
value=url_spec.hostname,
)
# Use the normalized URL to prevent parsing discrepancies
# between urllib3 and aiohttp (e.g. backslash-@ attacks).
url = url_spec.url
try:
return await global_http_connection.async_get_bytes(
url,
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
max_bytes=_get_audio_download_limit_bytes(),
)
except HTTPResponseSizeExceededError as exc:
_raise_audio_filesize_limit(exc.received_bytes)
else:
raise VLLMValidationError(
f"Unsupported URL scheme: {parsed.scheme}. "
"Supported schemes: http, https, data",
parameter="url",
value=parsed.scheme,
)
def make_error_request_output(
request: BatchRequestInput, error_msg: str
) -> BatchRequestOutput:
batch_output = BatchRequestOutput(
id=f"vllm-{random_uuid()}",
custom_id=request.custom_id,
response=BatchResponseData(
status_code=HTTPStatus.BAD_REQUEST,
request_id=f"vllm-batch-{random_uuid()}",
),
error=error_msg,
)
return batch_output
async def make_async_error_request_output(
request: BatchRequestInput, error_msg: str
) -> BatchRequestOutput:
return make_error_request_output(request, error_msg)
async def run_request(
serving_engine_func: Callable,
request: BatchRequestInput,
tracker: BatchProgressTracker,
) -> BatchRequestOutput:
try:
response = await serving_engine_func(request.body)
except Exception as e:
response = create_error_response(e)
if isinstance(response, JSONResponse):
with contextlib.suppress(pydantic.ValidationError):
response = TypeAdapter(AllResponse | ErrorResponse).validate_python(
json.loads(response.body)
)
if isinstance(response, AllResponse):
batch_output = BatchRequestOutput(
id=f"vllm-{random_uuid()}",
custom_id=request.custom_id,
response=BatchResponseData(
body=response, request_id=f"vllm-batch-{random_uuid()}"
),
error=None,
)
elif isinstance(response, ErrorResponse):
batch_output = BatchRequestOutput(
id=f"vllm-{random_uuid()}",
custom_id=request.custom_id,
response=BatchResponseData(
status_code=response.error.code,
request_id=f"vllm-batch-{random_uuid()}",
),
error=response,
)
else:
batch_output = make_error_request_output(
request, error_msg="Request must not be sent in stream mode"
)
tracker.completed()
return batch_output
WrapperFn: TypeAlias = Callable[[Callable], Callable]
def handle_endpoint_request(
request: BatchRequestInput,
tracker: BatchProgressTracker,
url_matcher: Callable[[str], bool],
handler_getter: Callable[[], Callable | None],
wrapper_fn: WrapperFn | None = None,
) -> Awaitable[BatchRequestOutput] | None:
"""Generic handler for endpoint requests.
Args:
request: The batch request input
tracker: Progress tracker for the batch
url_matcher: Function that takes a URL and returns True if it matches
handler_getter: Function that returns the handler function or None
wrapper_fn: Optional function to wrap the handler (e.g., for transcriptions)
Returns:
Awaitable[BatchRequestOutput] if the request was handled,
None if URL didn't match
"""
if not url_matcher(request.url):
return None
handler_fn = handler_getter()
if handler_fn is None:
error_msg = f"Model does not support endpoint: {request.url}"
return make_async_error_request_output(request, error_msg=error_msg)
# Apply wrapper if provided (e.g., for transcriptions/translations)
if wrapper_fn is not None:
handler_fn = wrapper_fn(handler_fn)
tracker.submitted()
return run_request(handler_fn, request, tracker)
async def build_endpoint_registry(
engine_client: EngineClient,
args: Namespace,
) -> dict[str, dict[str, Any]]:
"""Build the endpoint registry with all serving objects and handler configurations.
Args:
engine_client: The engine client
args: Command line arguments
Returns:
Dictionary mapping endpoint keys to their configurations
"""
supported_tasks = await engine_client.get_supported_tasks()
logger.info("Supported tasks: %s", supported_tasks)
# Create a state object to hold serving objects
state = State()
# Initialize all serving objects using init_app_state
# This provides full functionality including chat template processing,
# LoRA support, tool servers, etc.
await init_app_state(engine_client, state, args, supported_tasks)
# Get serving objects from state (defaulting to None if not set)
openai_serving_chat = getattr(state, "openai_serving_chat", None)
# Registry of endpoint configurations
endpoint_registry: dict[str, dict[str, Any]] = {
"completions": {
"url_matcher": lambda url: url == "/v1/chat/completions",
"handler_getter": lambda: (
openai_serving_chat.create_chat_completion
if openai_serving_chat is not None
else None
),
"wrapper_fn": None,
},
}
return endpoint_registry
def validate_run_batch_args(args):
valid_reasoning_parsers = ReasoningParserManager.list_registered()
if (
reasoning_parser := args.structured_outputs_config.reasoning_parser
) and reasoning_parser not in valid_reasoning_parsers:
raise KeyError(
f"invalid reasoning parser: {reasoning_parser} "
f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
)
async def run_batch(
engine_client: EngineClient,
args: Namespace,
) -> None:
endpoint_registry = await build_endpoint_registry(
engine_client=engine_client,
args=args,
)
tracker = BatchProgressTracker()
logger.info("Reading batch from %s...", args.input_file)
# Submit all requests in the file to the engine "concurrently".
response_futures: list[Awaitable[BatchRequestOutput]] = []
for request_json in (await read_file(args.input_file)).strip().split("\n"):
# Skip empty lines.
request_json = request_json.strip()
if not request_json:
continue
request = BatchRequestInput.model_validate_json(request_json)
# Use the last segment of the URL as the endpoint key.
# More advanced URL matching is done in url_matcher of endpoint_registry.
endpoint_key = request.url.split("/")[-1]
result = None
if endpoint_key in endpoint_registry:
endpoint_config = endpoint_registry[endpoint_key]
result = handle_endpoint_request(
request,
tracker,
url_matcher=endpoint_config["url_matcher"],
handler_getter=endpoint_config["handler_getter"],
wrapper_fn=endpoint_config["wrapper_fn"],
)
if result is not None:
response_futures.append(result)
else:
response_futures.append(
make_async_error_request_output(
request,
error_msg=f"URL {request.url} was used. "
"Supported endpoints: /v1/chat/completions.",
)
)
with tracker.pbar():
responses = await asyncio.gather(*response_futures)
await write_file(args.output_file, responses, args.output_tmp_dir)
async def main(args: Namespace):
from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client
from vllm.usage.usage_lib import UsageContext
validate_run_batch_args(args)
async with build_async_engine_client(
args,
usage_context=UsageContext.OPENAI_BATCH_RUNNER,
) as engine_client:
await run_batch(engine_client, args)
if __name__ == "__main__":
args = parse_args()
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
logger.info("args: %s", args)
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
# to publish metrics at the /metrics endpoint.
if args.enable_metrics:
logger.info("Prometheus metrics enabled")
start_http_server(port=args.port, addr=args.host)
else:
logger.info("Prometheus metrics disabled")
asyncio.run(main(args))
+1 -2
View File
@@ -30,7 +30,6 @@ from vllm.config.quantization import QuantizationConfigArgs
from vllm.engine.arg_utils import EngineArgs
from vllm.entrypoints.common.offline import _O, _R, OfflineInferenceMixin
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
from vllm.entrypoints.rl.offline import RLOfflineMixin
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
from vllm.inputs import PromptType
from vllm.logger import configure_logging_if_needed, init_logger
@@ -59,7 +58,7 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class LLM(BeamSearchOfflineMixin, RLOfflineMixin, OfflineInferenceMixin):
class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
"""An LLM for generating texts from given prompts and sampling parameters.
This class includes a tokenizer, a language model (possibly distributed
-4
View File
@@ -18,8 +18,6 @@ from vllm.entrypoints.launchers.launcher import (
setup_server,
validate_api_server_args,
)
from vllm.entrypoints.launchers.render.app_state import init_render_app_state
from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer
warnings.warn(
"`vllm.entrypoints.openai.api_server` is deprecated and will likely be"
@@ -35,13 +33,11 @@ __all__ = [
"build_async_engine_client_from_engine_args",
"build_app",
"init_app_state",
"init_render_app_state",
"create_server_socket",
"create_server_unix_socket",
"validate_api_server_args",
"setup_server",
"build_and_serve",
"build_and_serve_renderer",
"run_server",
"run_server_worker",
"register_api_routers",
@@ -107,7 +107,6 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
class ChatCompletionResponseChoice(OpenAIBaseModel):
index: int
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
# so that added fields like ``citations`` survive JSON serialization
# instead of being stripped down to the base schema. Plain
# ``ChatMessage`` instances serialize identically to before.
@@ -162,7 +161,6 @@ class ChatCompletionResponse(OpenAIBaseModel):
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
index: int
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
# so streaming ``citations`` survive JSON serialization. Plain
# ``DeltaMessage`` instances serialize identically to before.
delta: SerializeAsAny[DeltaMessage]
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,91 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Translate Decisions questions and answers to and from label reads."""
import json
from .protocol import (
ChoiceAnswer,
ChoiceQuestion,
DecisionAnswer,
DecisionInputMessage,
DecisionQuestion,
PredicateAnswer,
PredicateQuestion,
ScoreAnswer,
)
from .question_types import (
LABELS,
Option,
Question,
StructuredDecisionError,
argmax,
)
from .question_types import (
ChoiceQuestion as LabelQuestion,
)
def make_read_question(
index: int, question: DecisionQuestion, max_options: int = len(LABELS)
) -> Question:
if isinstance(question, PredicateQuestion):
options = [Option("false"), Option("true")]
elif isinstance(question, ChoiceQuestion):
options = [
Option(json.dumps(c.value, ensure_ascii=False), c.description)
for c in question.choices
]
else:
options = [Option(level.label, level.description) for level in question.levels]
limit = min(max_options, len(LABELS))
if len(options) > limit:
raise StructuredDecisionError(
f"This model supports at most {limit} choices per question"
)
return Question(
id=str(index),
type=LabelQuestion(),
instructions=question.instructions,
options=tuple(options),
labels=LABELS[: len(options)],
)
def make_answer(
question: DecisionQuestion, probs: list[float], label_mass: float
) -> DecisionAnswer:
if isinstance(question, PredicateQuestion):
return PredicateAnswer(name=question.name, probability=probs[1])
top = argmax(probs)
confidence = probs[top] * label_mass
if isinstance(question, ChoiceQuestion):
return ChoiceAnswer(
name=question.name,
choice=question.choices[top].value,
probabilities=[
{"value": choice.value, "probability": probability}
for choice, probability in zip(question.choices, probs)
],
confidence=confidence,
)
return ScoreAnswer(
name=question.name,
score=sum(i * probability for i, probability in enumerate(probs)),
probabilities=[
{"value": i, "label": level.label, "probability": probability}
for i, (level, probability) in enumerate(zip(question.levels, probs))
],
confidence=confidence,
)
def input_text(input: str | list[DecisionInputMessage]) -> str:
if isinstance(input, str):
return input
return "\n\n".join(
message.content
if isinstance(message.content, str)
else "\n".join(part.text for part in message.content)
for message in input
)
@@ -1,56 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import DecisionRequest, DecisionResponse
from .serving import OpenAIServingDecisions
router = APIRouter()
@router.post(
"/v1/decisions",
dependencies=[Depends(validate_json_request)],
response_model=DecisionResponse,
responses={
status.value: {"model": ErrorResponse}
for status in (
HTTPStatus.BAD_REQUEST,
HTTPStatus.NOT_FOUND,
HTTPStatus.INTERNAL_SERVER_ERROR,
HTTPStatus.NOT_IMPLEMENTED,
)
},
)
@with_cancellation
@load_aware_call
async def create_decision(request: DecisionRequest, raw_request: Request):
handler: OpenAIServingDecisions | None = getattr(
raw_request.app.state, "openai_serving_decisions", None
)
if handler is None:
error = create_error_response(
NotImplementedError("The model does not support the Decisions API")
)
return JSONResponse(content=error.model_dump(), status_code=error.error.code)
result = await handler.create_decisions(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def register_decisions_api_router(app: FastAPI):
app.include_router(router)
@@ -1,181 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""OpenAI Decisions API request and response models."""
from typing import Annotated, Any, Literal
from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
ShortText = Annotated[StrictStr, Field(max_length=1048576)]
InputText = Annotated[StrictStr, Field(max_length=10485760)]
ChoiceValue = StrictStr | StrictBool
class DecisionModel(OpenAIBaseModel):
model_config = ConfigDict(extra="forbid")
class DecisionInputText(DecisionModel):
type: Literal["input_text"]
text: InputText
class DecisionInputMessage(DecisionModel):
role: Literal["user"]
content: InputText | Annotated[list[DecisionInputText], Field(max_length=16384)]
type: Literal["message"] = "message"
@model_validator(mode="before")
@classmethod
def reject_images(cls, data: Any) -> Any:
if (
isinstance(data, dict)
and isinstance(data.get("content"), list)
and any(
isinstance(part, dict) and part.get("type") == "input_image"
for part in data["content"]
)
):
raise ValueError(
"Decisions supports text input only; images are not supported"
)
return data
class QuestionBase(DecisionModel):
instructions: ShortText
name: ShortText | None = None
@model_validator(mode="after")
def validate_name(self) -> "QuestionBase":
if "name" in self.model_fields_set and self.name is None:
raise ValueError("name must be a string when provided")
return self
class PredicateQuestion(QuestionBase):
type: Literal["predicate"]
class ChoiceOption(DecisionModel):
value: ChoiceValue
description: ShortText = ""
class ChoiceQuestion(QuestionBase):
type: Literal["choice"]
choices: list[ChoiceOption] = Field(min_length=2, max_length=255)
@model_validator(mode="after")
def validate_choices(self) -> "ChoiceQuestion":
values = [(type(c.value), c.value) for c in self.choices]
if len(set(values)) != len(values):
raise ValueError("choice values must be distinct")
return self
class ScoreLevel(DecisionModel):
label: ShortText
description: ShortText = ""
class ScoreQuestion(QuestionBase):
type: Literal["score"]
levels: list[ScoreLevel] = Field(min_length=2, max_length=10)
DecisionQuestion = Annotated[
PredicateQuestion | ChoiceQuestion | ScoreQuestion, Field(discriminator="type")
]
class DecisionRequest(DecisionModel):
model: ShortText
input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)]
questions: list[DecisionQuestion] = Field(min_length=1, max_length=200)
safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None
cache_salt: str | None = Field(
default=None,
min_length=1,
max_length=1024,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
if isinstance(data, dict):
validate_cache_salt(data.get("cache_salt"))
return data
class PredicateAnswer(DecisionModel):
type: Literal["predicate"] = "predicate"
name: str | None
probability: float
class ChoiceProbability(DecisionModel):
value: ChoiceValue
probability: float
class ChoiceAnswer(DecisionModel):
type: Literal["choice"] = "choice"
name: str | None
choice: ChoiceValue
probabilities: list[ChoiceProbability]
confidence: float
class ScoreProbability(DecisionModel):
value: int
label: str
probability: float
class ScoreAnswer(DecisionModel):
type: Literal["score"] = "score"
name: str | None
score: float
probabilities: list[ScoreProbability]
confidence: float
DecisionAnswer = Annotated[
PredicateAnswer | ChoiceAnswer | ScoreAnswer,
Field(discriminator="type"),
]
class InputTokensDetails(DecisionModel):
cached_tokens: int = 0
cache_write_tokens: int = 0
class OutputTokensDetails(DecisionModel):
reasoning_tokens: int = 0
class OpenAIDecisionUsage(DecisionModel):
input_tokens: int
input_tokens_details: InputTokensDetails
output_tokens: int
output_tokens_details: OutputTokensDetails
total_tokens: int
class DecisionResponse(DecisionModel):
model: str
answers: list[DecisionAnswer]
usage: OpenAIDecisionUsage
@@ -1,240 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Question types for structured decisions.
A question type turns a request's criteria into options, writes the question
for the model with a label per option, and builds the answer from the label
probabilities.
"""
import math
import string
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar
class StructuredDecisionError(ValueError):
"""An invalid request. Returned as a 400."""
#: A choice labels its options A to Z in order. A single capital letter is one
#: token at the start of a reply for the tokenizers tested. Every read checks it.
LABELS = tuple(string.ascii_uppercase)
@dataclass(frozen=True)
class Option:
name: str
description: str | None = None
@dataclass(frozen=True)
class Question:
id: str
type: "QuestionType"
instructions: str
options: tuple[Option, ...]
labels: tuple[str, ...]
class QuestionType(ABC):
name: ClassVar[str]
#: The labels of a question's options, in option order.
label_set: ClassVar[tuple[str, ...]] = LABELS
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
@abstractmethod
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
@abstractmethod
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
"""The answer for ``question``. ``probs[i]`` is the probability of
``question.labels[i]`` among the labels, and the list sums to 1.
``label_mass`` is the labels' total probability over the vocabulary."""
def reply_line(self, label: str, option: Option) -> str:
return (
f"{label}: {option.name} - {option.description}"
if option.description
else f"{label}: {option.name}"
)
def prompt(self, question: Question) -> str:
"""The question as the model reads it, after the state."""
lines = [f"Question: {question.instructions}"] if question.instructions else []
for label, o in zip(question.labels, question.options):
lines.append(self.reply_line(label, o))
lines.append(self.reply_instruction)
return "\n".join(lines)
QUESTION_TYPES: dict[str, QuestionType] = {}
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
if cls.name in QUESTION_TYPES:
raise ValueError(f"question type {cls.name!r} is already registered")
QUESTION_TYPES[cls.name] = cls()
return cls
def get_question_type(name: str) -> QuestionType:
try:
return QUESTION_TYPES[name]
except KeyError:
raise StructuredDecisionError(
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
) from None
def build_question(
qid: str,
type_name: str,
instructions: Any,
criteria: Any,
max_options: int,
labeled: bool = True,
) -> Question:
if not qid:
raise StructuredDecisionError("question ids must be non-empty")
qtype = get_question_type(type_name)
options = qtype.parse_options(qid, criteria)
if not options:
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
names = [o.name for o in options]
if len(set(names)) != len(names):
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
limit = min(max_options, len(qtype.label_set)) if labeled else max_options
if len(options) > limit:
raise StructuredDecisionError(
f"question {qid!r}: at most {limit} options for this model"
)
if not isinstance(instructions, str):
instructions = "" if instructions is None else str(instructions)
return Question(
id=qid,
type=qtype,
instructions=instructions,
options=tuple(options),
labels=qtype.label_set[: len(options)],
)
def argmax(values: list[float]) -> int:
return max(range(len(values)), key=values.__getitem__)
def label_softmax(logprobs: list[float]) -> list[float]:
"""Softmax of the label logprobs."""
top = max(logprobs)
weights = [math.exp(lp - top) for lp in logprobs]
total = sum(weights)
return [w / total for w in weights]
@register_question_type
class ChoiceQuestion(QuestionType):
"""Pick one option. ``criteria`` maps each option name to a description,
or to null."""
name = "choice"
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, dict) or not criteria:
raise StructuredDecisionError(
f"question {qid!r}: choice criteria must map option names to "
"a description or null"
)
return [
Option(str(name), None if desc is None else str(desc))
for name, desc in criteria.items()
]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
"choice": question.options[top].name,
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
# The chosen label's probability over the whole vocabulary, so a
# read that mostly wanted a non-label token reports low confidence.
"confidence": probs[top] * label_mass,
}
@register_question_type
class NoulQuestion(QuestionType):
"""Yes or no. ``criteria`` may describe what true and false mean."""
name = "noul"
label_set = ("yes", "no")
reply_instruction = "Answer with yes or no only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if criteria is not None and (
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
):
raise StructuredDecisionError(
f"question {qid!r}: noul criteria must be an object with true and false"
)
criteria = criteria or {}
true = criteria.get("true")
false = criteria.get("false")
return [
Option("yes", None if true is None else str(true)),
Option("no", None if false is None else str(false)),
]
def reply_line(self, label: str, option: Option) -> str:
# A side without a description still shows its label. Without the
# label in the prompt, models answer "Yes" or "No" instead.
return f"{label}: {option.description}" if option.description else label
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
return {
"type": self.name,
"noul": probs[0],
"probabilities": {"yes": probs[0], "no": probs[1]},
# The chosen label's probability over the whole vocabulary, like a
# choice's confidence.
"confidence": max(probs) * label_mass,
}
@register_question_type
class ScoreQuestion(QuestionType):
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
level names, from the first to the last level of the scale."""
name = "score"
# 0-indexed like the answer, so a level's label is its score.
label_set = tuple(string.digits)
reply_instruction = "Answer with the number of one level only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, list) or len(criteria) < 2:
raise StructuredDecisionError(
f"question {qid!r}: score criteria must be an ordered list of levels"
)
return [Option(str(level)) for level in criteria]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
# The expected level, 0-indexed, so a level's index in the request's
# criteria is its value.
"score": sum(i * p for i, p in enumerate(probs)),
"legend": {str(i): o.name for i, o in enumerate(question.options)},
"probabilities": {str(i): p for i, p in enumerate(probs)},
"confidence": probs[top] * label_mass,
}
@@ -1,102 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from fastapi import Request
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from .adapters import input_text, make_answer, make_read_question
from .protocol import (
DecisionRequest,
DecisionResponse,
InputTokensDetails,
OpenAIDecisionUsage,
OutputTokensDetails,
)
from .question_types import StructuredDecisionError
from .strategies import ReadStrategy
class BaseServingDecisions(BaseServing):
def __init__(
self,
models: OpenAIServingModels,
strategy: ReadStrategy,
*,
request_logger: RequestLogger | None = None,
) -> None:
super().__init__(
models=models,
model_config=strategy.context.engine_client.model_config,
request_logger=request_logger,
)
self.strategy = strategy
self.limits = strategy.limits()
class OpenAIServingDecisions(BaseServingDecisions):
async def create_decisions(
self, request: DecisionRequest, raw_request: Request | None = None
) -> DecisionResponse | ErrorResponse:
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
return error
engine = self.strategy.context.engine_client
if engine.errored:
raise engine.dead_error
request_id = f"decision-{self._base_request_id(raw_request)}"
try:
if len(request.questions) > self.limits.max_questions:
raise StructuredDecisionError(
f"questions: at most {self.limits.max_questions} for this model"
)
questions = [
make_read_question(i, question, self.limits.max_options)
for i, question in enumerate(request.questions)
]
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
engine.check_admission(len(questions))
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
text = input_text(request.input)
self._log_inputs(request_id, text, None, lora_request)
reads = await self.strategy.read(
questions,
None,
text,
request_id=request_id,
chat_template_kwargs=None,
lora_request=lora_request,
priority=0,
cache_salt=request.cache_salt,
trace_headers=trace_headers,
)
except StructuredDecisionError as exc:
return self.create_error_response(exc)
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
input_tokens = sum(read.input_tokens for read in reads)
output_tokens = sum(read.output_tokens for read in reads)
return DecisionResponse(
model=self.models.model_name(lora_request),
answers=[
make_answer(question, read.probs, read.label_mass)
for question, read in zip(request.questions, reads)
],
usage=OpenAIDecisionUsage(
input_tokens=input_tokens,
output_tokens=output_tokens,
total_tokens=input_tokens + output_tokens,
input_tokens_details=InputTokensDetails(
cached_tokens=sum(read.cached_tokens for read in reads),
cache_write_tokens=sum(read.cache_write_tokens for read in reads),
),
output_tokens_details=OutputTokensDetails(),
),
)
@@ -1,46 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from typing import Any
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.tasks import SupportedTask
from .serving import OpenAIServingDecisions
from .strategies import ReadContext, ReadStrategy, select_read_strategy
def init_decisions_state(
engine_client: EngineClient,
state: State,
args: Namespace,
request_logger: RequestLogger | None,
chat_template: str | None,
default_chat_template_kwargs: dict[str, Any],
supported_tasks: tuple[SupportedTask, ...],
) -> ReadStrategy | None:
state.openai_serving_decisions = None
try:
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
strategy = strategy_cls(
ReadContext(
engine_client=engine_client,
online_renderer=state.online_renderer,
chat_template=chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=default_chat_template_kwargs,
)
)
except ValueError:
return None
if strategy.task == "generate":
state.openai_serving_decisions = OpenAIServingDecisions(
state.openai_serving_models,
strategy,
request_logger=request_logger,
)
return strategy
File diff suppressed because it is too large Load Diff
-37
View File
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import warnings
from prometheus_client import start_http_server
from vllm.logger import init_logger
from vllm.version import __version__ as VLLM_VERSION
warnings.warn(
"The `python -m vllm.entrypoints.openai.run_batch` command is deprecated "
"and may be removed in a future release. Please use `vllm run_batch` instead.",
DeprecationWarning,
stacklevel=1,
)
if __name__ == "__main__":
from vllm.entrypoints.launchers.run_batch import main, parse_args
logger = init_logger(__name__)
args = parse_args()
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
logger.info("args: %s", args)
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
# to publish metrics at the /metrics endpoint.
if args.enable_metrics:
logger.info("Prometheus metrics enabled")
start_http_server(port=args.port, addr=args.host)
else:
logger.info("Prometheus metrics disabled")
asyncio.run(main(args))
View File
-11
View File
@@ -1,11 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import FastAPI
def register_rl_api_routers(app: FastAPI):
from .online.api_router import router as rl_router
app.include_router(rl_router)
-68
View File
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
WeightTransferUpdateRequest,
)
from vllm.entrypoints.common.offline import OfflineInferenceMixin
class RLOfflineMixin(OfflineInferenceMixin):
def init_weight_transfer_engine(
self, request: WeightTransferInitRequest | dict
) -> None:
"""Initialize weight transfer for RL training.
Args:
request: Weight transfer initialization request with backend-specific info
"""
init_info_dict = (
request["init_info"] if isinstance(request, dict) else request.init_info
)
self.llm_engine.collective_rpc(
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
)
def shutdown_weight_transfer_engine(self) -> None:
"""Release the trainer connection without shutting down inference."""
self.llm_engine.collective_rpc("shutdown_weight_transfer_engine")
def start_weight_update(self) -> None:
"""Start a new weight update."""
self.llm_engine.collective_rpc("start_weight_update")
def start_draft_weight_update(self) -> None:
"""Start a new weight update targeting the speculative draft model."""
self.llm_engine.collective_rpc("start_draft_weight_update")
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
"""Update the weights of the model.
Args:
request: Weight update request with backend-specific update info
"""
update_info_dict = (
request["update_info"] if isinstance(request, dict) else request.update_info
)
self.llm_engine.collective_rpc(
"update_weights", kwargs={"update_info": update_info_dict}
)
def finish_weight_update(self, weight_version: str | None = None) -> None:
"""Finish the weight update and set its version if provided."""
self.llm_engine.collective_rpc("finish_weight_update")
if weight_version is not None:
self.llm_engine.set_weight_version(weight_version)
def update_weight_version(self, new_version: str) -> None:
"""Set the weight version without updating weights."""
self.llm_engine.set_weight_version(new_version)
def get_weight_version(self) -> str:
"""Return the latest committed weight version."""
return self.llm_engine.get_weight_version()
-286
View File
@@ -1,286 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from http import HTTPStatus
from typing import Annotated, Any, Literal
from fastapi import APIRouter, Body, HTTPException, Query, Request
from fastapi.responses import JSONResponse
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
WeightTransferUpdatePayload,
WeightTransferUpdateRequest,
)
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.rl.online.metrics import weight_operation_metrics
from vllm.logger import init_logger
from vllm.v1.engine import PauseMode
logger = init_logger(__name__)
def engine_client(request: Request) -> EngineClient:
return request.app.state.engine_client
router = APIRouter()
@router.post("/pause")
async def pause_generation(
raw_request: Request,
mode: Annotated[PauseMode, Query()] = "abort",
clear_cache: Annotated[bool, Query()] = True,
) -> JSONResponse:
"""Pause generation requests to allow weight updates.
Args:
raw_request: The incoming FastAPI request, used to reach the engine
client on the app state.
mode: How to handle in-flight requests:
- ``"abort"``: Abort all in-flight requests immediately (default).
- ``"wait"``: Wait for in-flight requests to complete.
- ``"keep"``: Freeze requests in queue; they resume on /resume.
clear_cache: DEPRECATED. Whether to clear KV/prefix caches after
draining. Ignored when mode="keep".
"""
engine = engine_client(raw_request)
try:
await engine.pause_generation(
mode=mode,
clear_cache=clear_cache,
)
return JSONResponse(
content={"status": "paused"},
status_code=HTTPStatus.OK.value,
)
except ValueError as err:
return JSONResponse(
content={"error": str(err)},
status_code=HTTPStatus.BAD_REQUEST.value,
)
except Exception as err: # pragma: no cover - defensive
logger.exception("Failed to pause generation")
return JSONResponse(
content={"error": f"Failed to pause generation: {err}"},
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
)
@router.post("/resume")
async def resume_generation(raw_request: Request) -> JSONResponse:
"""Resume generation after a pause."""
engine = engine_client(raw_request)
try:
await engine.resume_generation()
return JSONResponse(
content={"status": "resumed"},
status_code=HTTPStatus.OK.value,
)
except Exception as err: # pragma: no cover - defensive
logger.exception("Failed to resume generation")
return JSONResponse(
content={"error": f"Failed to resume generation: {err}"},
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
)
@router.post("/abort_requests")
async def abort_requests(raw_request: Request) -> JSONResponse:
"""Abort in-flight requests without pausing the scheduler.
Empty/missing ``request_ids`` aborts all in-flight requests.
"""
engine = engine_client(raw_request)
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904
if not isinstance(body, dict):
raise HTTPException(
status_code=400, detail="Request body must be a JSON object"
)
request_ids = body.get("request_ids")
try:
if request_ids:
# Body ids are external (user-supplied) request ids.
await engine.abort(request_ids)
else:
# The dev RL server runs AsyncLLM; abort everything it is tracking.
# request_states is keyed by internal ids; parent_requests holds
# parallel-sampling parents. Abort both as internal ids.
from vllm.v1.engine.async_llm import AsyncLLM
assert isinstance(engine, AsyncLLM)
op = engine.output_processor
request_ids = [
*op.request_states.keys(),
*op.parent_requests.keys(),
]
await engine.abort(request_ids, internal=True)
return JSONResponse(
content={"status": "aborted", "aborted": len(request_ids)},
status_code=HTTPStatus.OK.value,
)
except Exception as err: # pragma: no cover - defensive
logger.exception("Failed to abort requests")
return JSONResponse(
content={"error": f"Failed to abort requests: {err}"},
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
)
@router.get("/is_paused")
async def is_paused(raw_request: Request) -> JSONResponse:
"""Return the current pause status."""
engine = engine_client(raw_request)
try:
paused = await engine.is_paused()
except Exception as err: # pragma: no cover - defensive
logger.exception("Failed to fetch pause status")
return JSONResponse(
content={"error": f"Failed to fetch pause status: {err}"},
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
)
return JSONResponse(content={"is_paused": paused})
@router.post("/init_weight_transfer_engine")
async def init_weight_transfer_engine(
raw_request: Request,
init_info: Annotated[dict[str, Any], Body(embed=True)],
):
with weight_operation_metrics().record("init"):
await engine_client(raw_request).init_weight_transfer_engine(
WeightTransferInitRequest(init_info=init_info)
)
return JSONResponse(content={"message": "Weight transfer initialized"})
@router.post("/shutdown_weight_transfer_engine")
async def shutdown_weight_transfer_engine(raw_request: Request):
with weight_operation_metrics().record("shutdown"):
await engine_client(raw_request).shutdown_weight_transfer_engine()
return JSONResponse(content={"message": "Weight transfer shut down"})
@router.post("/start_weight_update")
async def start_weight_update(raw_request: Request):
with weight_operation_metrics().record("start"):
await engine_client(raw_request).start_weight_update()
return JSONResponse(content={"message": "Weight update started"})
@router.post("/start_draft_weight_update")
async def start_draft_weight_update(raw_request: Request):
with weight_operation_metrics().record("start_draft"):
await engine_client(raw_request).start_draft_weight_update()
return JSONResponse(content={"message": "Draft weight update started"})
@router.post("/update_weights")
async def update_weights(
raw_request: Request,
update_info: Annotated[WeightTransferUpdatePayload, Body(embed=True)],
):
with weight_operation_metrics().record("update"):
await engine_client(raw_request).update_weights(
request=WeightTransferUpdateRequest(update_info=update_info)
)
return JSONResponse(content={"message": "Weights updated"})
@router.post("/finish_weight_update")
async def finish_weight_update(
raw_request: Request,
weight_version: Annotated[str | None, Body(embed=True)] = None,
):
with weight_operation_metrics().record("finish"):
await engine_client(raw_request).finish_weight_update(weight_version)
return JSONResponse(content={"message": "Weight update finished"})
@router.post("/update_weight_version")
async def update_weight_version(
raw_request: Request,
new_version: Annotated[str, Body(embed=True)],
):
await engine_client(raw_request).update_weight_version(new_version)
return JSONResponse(content={"success": True, "new_version": new_version})
@router.get("/weight_info")
async def weight_info(raw_request: Request):
weight_version = await engine_client(raw_request).get_weight_version()
return JSONResponse(content={"weight_version": weight_version})
@router.post("/weight_checker")
async def weight_checker(
raw_request: Request,
action: Annotated[Literal["checksum", "reset", "compare"], Body(embed=True)],
baseline: Annotated[dict[str, str] | None, Body(embed=True)] = None,
) -> JSONResponse:
"""Checksum, reset, or compare model weights against a baseline."""
client = engine_client(raw_request)
if action == "reset":
await client.collective_rpc("reset_weights")
return JSONResponse(content={"status": "reset"})
if action == "compare" and baseline is None:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="action='compare' requires a 'baseline' object",
)
checksums: dict[str, str] = {}
for worker_checksums in await client.compute_weight_checksums():
if duplicates := checksums.keys() & worker_checksums.keys():
raise HTTPException(
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
detail=f"Duplicate weight checksum keys: {sorted(duplicates)}",
)
checksums.update(worker_checksums)
if action == "checksum":
return JSONResponse(content={"checksums": checksums})
assert baseline is not None
mismatches = sorted(
key
for key in checksums.keys() | baseline.keys()
if checksums.get(key) != baseline.get(key)
)
return JSONResponse(content={"match": not mismatches, "mismatches": mismatches})
@router.get("/get_world_size")
async def get_world_size(
raw_request: Request,
include_dp: bool = Query(True),
):
"""Get the world size from the parallel config.
Args:
raw_request: The incoming FastAPI request, used to reach the engine
client on the app state.
include_dp: If True (default), returns the world size including
data parallelism (TP * PP * DP). If False, returns the world
size without data parallelism (TP * PP).
"""
parallel_config = engine_client(raw_request).vllm_config.parallel_config
if include_dp:
world_size = parallel_config.world_size_across_dp
else:
world_size = parallel_config.world_size
return JSONResponse(content={"world_size": world_size})
-56
View File
@@ -1,56 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Duration and in-flight metrics for RL weight-transfer operations."""
from collections.abc import Iterator
from contextlib import contextmanager
from threading import Lock
from time import perf_counter
from typing import Literal
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
Operation = Literal["init", "shutdown", "start", "start_draft", "update", "finish"]
class WeightOperationMetrics:
def __init__(self, registry: CollectorRegistry = REGISTRY):
self.duration = Histogram(
"vllm:rl_weight_update_operation_duration_seconds",
"Duration of one weight-transfer operation.",
["operation"],
buckets=(0.01, 0.1, 1, 10, 30, 60, 120, 300, 600),
registry=registry,
)
self.in_flight = Gauge(
"vllm:rl_weight_update_operations_in_flight",
"Weight-transfer operations currently awaited.",
["operation"],
multiprocess_mode="livesum",
registry=registry,
)
@contextmanager
def record(self, operation: Operation) -> Iterator[None]:
in_flight = self.in_flight.labels(operation)
in_flight.inc()
started = perf_counter()
try:
yield
finally:
in_flight.dec()
self.duration.labels(operation).observe(perf_counter() - started)
_metrics: WeightOperationMetrics | None = None
_metrics_lock = Lock()
def weight_operation_metrics() -> WeightOperationMetrics:
# Lazy, so the collectors exist only after multiprocess Prometheus setup.
global _metrics
if _metrics is None:
with _metrics_lock:
if _metrics is None:
_metrics = WeightOperationMetrics()
return _metrics
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
from ..token_in_token_out.protocol import (
DerenderChatRequestUnion,
DerenderChatStreamRequest,
DerenderChatStreamResponse,
DerenderCompletionRequestUnion,
DerenderCompletionStreamRequest,
DerenderCompletionStreamResponse,
)
from .serving import ServingDerender
logger = init_logger(__name__)
router = APIRouter()
def derender(request: Request) -> ServingDerender | None:
return getattr(request.app.state, "serving_derender", None)
@router.post(
"/v1/chat/completions/derender",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_chat_completion(
request: DerenderChatRequestUnion,
raw_request: Request,
):
"""Derender a generate response into a ChatCompletionResponse.
Accepts both non-streaming (``stream=false``, default) and streaming
(``stream=true``) request bodies on the same path; FastAPI validates and
routes on the ``stream`` discriminator.
Non-streaming: body is ``DerenderChatRequest`` (``generate_response`` with
the complete token list). Returns a ``ChatCompletionResponse``.
Streaming: body is ``DerenderChatStreamRequest`` (one ``generate_chunk``
delta + optional ``stream_state``). Returns a ``DerenderChatStreamResponse``
(``chunk`` + ``stream_state``). The client carries ``stream_state`` between
successive calls, one per SSE chunk from ``/inference/v1/generate``.
"""
handler = derender(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Derender API"
)
if isinstance(request, DerenderChatStreamRequest):
stream_result = await handler.derender_chat_stream_response(request)
if isinstance(stream_result, ErrorResponse):
return JSONResponse(
content=stream_result.model_dump(),
status_code=stream_result.error.code,
)
chunk, stream_state = stream_result
response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
return JSONResponse(content=response.model_dump())
result = await handler.derender_chat_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/derender",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_completion(
request: DerenderCompletionRequestUnion,
raw_request: Request,
):
"""Derender a generate response into a CompletionResponse.
Accepts both non-streaming (``stream=false``, default) and streaming
(``stream=true``) request bodies on the same path.
Non-streaming: body is ``DerenderCompletionRequest``. Returns a
``CompletionResponse``.
Streaming: body is ``DerenderCompletionStreamRequest`` (one
``generate_chunk`` + optional ``stream_state``). Returns a
``DerenderCompletionStreamResponse`` (``chunk`` + ``stream_state``).
"""
handler = derender(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Derender API")
if isinstance(request, DerenderCompletionStreamRequest):
stream_result = await handler.derender_completion_stream_response(request)
if isinstance(stream_result, ErrorResponse):
return JSONResponse(
content=stream_result.model_dump(),
status_code=stream_result.error.code,
)
chunk, stream_state = stream_result
response = DerenderCompletionStreamResponse(
chunk=chunk, stream_state=stream_state
)
return JSONResponse(content=response.model_dump())
result = await handler.derender_completion_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@@ -1,434 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
import vllm.envs as envs
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse,
CompletionStreamResponse,
)
from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.renderers.online_derenderer import OnlineDerenderer
from ..token_in_token_out.mm_features import extract_mm_features
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
DerenderStreamState,
GenerateTokensResponse,
MultiModalFeatures,
)
logger = init_logger(__name__)
class ServingDerender(BaseServing):
def __init__(
self,
models: OpenAIServingModels | OpenAIModelRegistry,
online_derenderer: "OnlineDerenderer",
*,
request_logger: RequestLogger | None = None,
) -> None:
super().__init__(
models=models,
model_config=models.model_config,
request_logger=request_logger,
)
self.online_derenderer = online_derenderer
def _validate_derender_bounds(
self,
generate_responses: list[GenerateTokensResponse],
) -> ErrorResponse | None:
"""Reject derender payloads that exceed resource bounds.
Runs before any tokenizer.decode() or parser invocation to prevent
CPU/memory exhaustion from oversized caller-supplied token structures.
"""
max_n = envs.VLLM_MAX_N_SEQUENCES
max_model_len = self.model_config.max_model_len
# See ModelConfig.max_logprobs for semantics and default value.
max_logprobs = self.model_config.max_logprobs
if len(generate_responses) > max_n:
return self.create_error_response(
f"generate_responses count ({len(generate_responses)}) "
f"exceeds server maximum ({max_n}). "
f"Set VLLM_MAX_N_SEQUENCES to increase this limit."
)
for gen in generate_responses:
if len(gen.choices) > max_n:
return self.create_error_response(
f"choices count ({len(gen.choices)}) in response "
f"'{gen.request_id}' exceeds server maximum ({max_n})."
)
for choice in gen.choices:
if choice.token_ids and len(choice.token_ids) > max_model_len:
return self.create_error_response(
f"token_ids length ({len(choice.token_ids)}) in "
f"choice {choice.index} exceeds "
f"max_model_len ({max_model_len})."
)
if choice.logprobs and choice.logprobs.content:
if len(choice.logprobs.content) > max_model_len:
return self.create_error_response(
f"logprobs.content length "
f"({len(choice.logprobs.content)}) in "
f"choice {choice.index} exceeds "
f"max_model_len ({max_model_len})."
)
for entry in choice.logprobs.content:
if (
max_logprobs >= 0
and entry.top_logprobs
and len(entry.top_logprobs) > max_logprobs
):
return self.create_error_response(
f"top_logprobs count "
f"({len(entry.top_logprobs)}) in "
f"choice {choice.index} exceeds "
f"max_logprobs ({max_logprobs})."
)
if gen.prompt_logprobs and len(gen.prompt_logprobs) > max_model_len:
return self.create_error_response(
f"prompt_logprobs length ({len(gen.prompt_logprobs)}) "
f"in response '{gen.request_id}' exceeds "
f"max_model_len ({max_model_len})."
)
return None
def _validate_prompt_token_ids(
self,
prompt_token_ids: list[list[int] | None],
) -> ErrorResponse | None:
"""Reject caller-supplied prompt_token_ids longer than max_model_len."""
max_model_len = self.model_config.max_model_len
for ids in prompt_token_ids:
if ids is not None and len(ids) > max_model_len:
return self.create_error_response(
f"prompt_token_ids length ({len(ids)}) exceeds "
f"max_model_len ({max_model_len})."
)
return None
async def derender_chat_response(
self,
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
bounds_error = self._validate_derender_bounds([request.generate_response])
if bounds_error is not None:
return bounds_error
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
if prompt_ids_error is not None:
return prompt_ids_error
if self.online_derenderer.parser is not None and request.chat_request is None:
return self.create_error_response(
"chat_request is required when a tool or reasoning parser is "
"configured because plain detokenization would leak raw parser "
"markup into content."
)
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response,
request.chat_request,
request.prompt_token_ids,
)
except ValueError as exc:
return self.create_error_response(str(exc))
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
gen = request.generate_response
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
model_name = request.model or self.models.model_name()
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
model_name,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=model_name,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
metrics=gen.metrics,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
bounds_error = self._validate_derender_bounds(request.generate_responses)
if bounds_error is not None:
return bounds_error
if request.prompt_token_ids is not None:
prompt_ids_error = self._validate_prompt_token_ids(request.prompt_token_ids)
if prompt_ids_error is not None:
return prompt_ids_error
(
choices,
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses,
request.prompt_tokens,
completion_request=request.completion_request,
prompt_token_ids=request.prompt_token_ids,
)
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
model_name = request.model or self.models.model_name()
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
model_name,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=model_name,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
# Metrics describe one prompt. Parallel samples are already
# suppressed by /generate; multi-prompt responses cannot be merged.
metrics=first.metrics if len(request.generate_responses) == 1 else None,
)
async def derender_chat_stream_response(
self,
request: DerenderChatStreamRequest,
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState] | ErrorResponse:
"""Streaming counterpart to ``derender_chat_response``.
Processes one ``GenerateStreamResponse`` chunk and returns the
derendered chunk together with the updated client carried state.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if self.online_derenderer.parser is not None and request.chat_request is None:
return self.create_error_response(
"chat_request is required when a tool or reasoning parser is "
"configured because plain detokenization would leak raw parser "
"markup into content."
)
if (
self.online_derenderer.parser is not None
and request.prompt_token_ids is None
):
return self.create_error_response(
"prompt_token_ids is required when a tool or reasoning "
"parser is configured. Without it parse_delta cannot tell "
"whether the prompt left reasoning open (e.g. a chat "
"template that pre-opens <think>) and would misclassify "
"reasoning content as plain content."
)
# Each streamed chunk contains at most one choice per SSE event.
# A single DerenderStreamState is threaded
# through every choice in the chunk, so >1 would corrupt detok/parser
# state across choices. See the matching check in
# OnlineDerenderer.derender_chat_stream which this fails ahead of to
# avoid touching the tokenizer at all on a malformed chunk.
num_choices = len(request.generate_chunk.choices)
if num_choices > 1:
return self.create_error_response(
f"derender_chat_stream expects at most one choice per chunk "
f"(got {num_choices})."
)
# prompt_token_ids is folded in here too. It is caller supplied and
# otherwise unbounded. A parser configured deployment rescans it
# in full on every chunk (is_reasoning_end /
# adjust_initial_state_from_prompt, see abstract_parser.parse_delta)
# since parser state cannot be carried across calls.
prompt_len = (
len(request.prompt_token_ids) if request.prompt_token_ids is not None else 0
)
prior_len = (
len(request.stream_state.output_token_ids)
if request.stream_state is not None
else 0
)
delta_len = sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
)
max_model_len = self.model_config.max_model_len
if prompt_len + prior_len + delta_len > max_model_len:
return self.create_error_response(
f"prompt_token_ids length ({prompt_len}) plus "
f"output_token_ids length ({prior_len}) plus delta "
f"({delta_len}) exceeds max_model_len ({max_model_len})."
)
model_name = request.model or self.models.model_name()
try:
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
model=model_name,
generate_chunk=request.generate_chunk,
state=request.stream_state,
chat_request=request.chat_request,
prompt_tokens=request.prompt_tokens,
prompt_token_ids=request.prompt_token_ids,
)
except Exception as exc:
# The two ValueErrors derender_chat_stream can raise directly
# (missing chat_request, >1 choice per chunk) are already
# pre-checked above, so anything reaching here comes from the
# parser replaying entirely client controlled state. Hermes
# swallows its own streaming exceptions but finalize_generation
# and engine based parsers don't, so an unexpected parser
# failure on malformed (but well typed) state must surface as
# a 400 here rather than an unhandled 500.
logger.debug(
"derender stream replay failed: %s: %s", type(exc).__name__, exc
)
return self.create_error_response("invalid stream_state: derender failed")
logger.debug(
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
model_name,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
)
return chunk, updated_state
async def derender_completion_stream_response(
self,
request: DerenderCompletionStreamRequest,
) -> tuple[CompletionStreamResponse, DerenderStreamState] | ErrorResponse:
"""Streaming counterpart to ``derender_completion_response``.
Processes one ``GenerateStreamResponse`` chunk (one output sequence's
delta) and returns the derendered chunk and updated state.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
if prompt_ids_error is not None:
return prompt_ids_error
model_name = request.model or self.models.model_name()
try:
(
chunk,
updated_state,
) = await self.online_derenderer.derender_completion_stream(
model=model_name,
generate_chunk=request.generate_chunk,
state=request.stream_state,
prompt_tokens=request.prompt_tokens,
completion_request=request.completion_request,
prompt_token_ids=request.prompt_token_ids,
)
except ValueError as exc:
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc})"
)
logger.debug(
"derender_completion_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
model_name,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
)
return chunk, updated_state
@staticmethod
def _extract_mm_features(
engine_input: EngineInput,
) -> MultiModalFeatures | None:
return extract_mm_features(engine_input)
+4 -38
View File
@@ -25,35 +25,12 @@ else:
logger = init_logger(__name__)
def init_render_state(
state: "State",
request_logger: RequestLogger | None,
):
from .derender.serving import ServingDerender
from .render.serving import ServingRender
state.serving_render = ServingRender(
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
tool_server=state.tool_server,
)
state.serving_derender = ServingDerender(
state.openai_serving_models,
state.online_derenderer,
request_logger=request_logger,
)
def init_scale_out_state(
state: "State",
args: "Namespace",
engine_client: "EngineClient",
request_logger: RequestLogger | None,
):
init_render_state(state, request_logger)
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
state.serving_tokens = ServingTokens(
@@ -72,14 +49,10 @@ def register_scale_out_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
):
from .derender.api_router import router as derender_render
from .render.api_router import router as render_render
args = getattr(app.state, "args", None)
# `vllm launch render` and `vllm serve --tokens-only` are dedicated to
# serving these endpoints, so they always register them regardless of
# `--enable-scale-out`.
dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False)
# `vllm serve --tokens-only` is dedicated to serving these endpoints, so it
# always registers them regardless of `--enable-scale-out`.
dedicated_mode = getattr(args, "tokens_only", False)
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
if not enabled:
message = (
@@ -96,11 +69,7 @@ def register_scale_out_api_routers(
from .token_in_token_out.api_router import router as token_router
for route in (
*render_render.routes,
*derender_render.routes,
*token_router.routes,
):
for route in token_router.routes:
if isinstance(route, APIRoute):
app.add_api_route(
route.path,
@@ -110,9 +79,6 @@ def register_scale_out_api_routers(
)
return
app.include_router(render_render)
app.include_router(derender_render)
if "generate" in supported_tasks:
from .token_in_token_out.api_router import (
attach_router as attach_disagg_router,
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
from ..token_in_token_out.protocol import GenerateRequest
from .serving import ServingRender
logger = init_logger(__name__)
router = APIRouter()
def render(request: Request) -> ServingRender | None:
return getattr(request.app.state, "serving_render", None)
@router.post(
"/v1/chat/completions/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Render API"
)
result = await handler.render_chat_request(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/messages/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_messages(request: AnthropicMessagesRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Messages Render API")
result = await handler.render_messages_request(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/render",
dependencies=[Depends(validate_json_request)],
response_model=list[GenerateRequest],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_completion(request: CompletionRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Render API")
result = await handler.render_completion_request(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=[item.model_dump() for item in result])
@router.post(
"/v1/responses/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_responses(request: ResponsesRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Responses Render API")
result = await handler.render_responses_request(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@@ -1,385 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
from fastapi import Request
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.generate.base.protocol import MMPayload
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
MultiModalFeatures,
ReasoningParserKwargs,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.multimodal.parse import MultiModalDataParser
from vllm.renderers.inputs.preprocess import (
extract_prompt_components,
extract_prompt_len,
)
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
logger = init_logger(__name__)
if TYPE_CHECKING:
from vllm.entrypoints.mcp.tool_server import ToolServer
def _hoist_transfer_params(
params: SamplingParams,
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
"""Move KV/EC transfer params out of `extra_args`.
Render emits them only as top-level `GenerateRequest` fields. Generate
copies them back into `extra_args` for the engine.
"""
if not params.extra_args:
return None, None
# to_sampling_params aliases request.vllm_xargs, so copy before popping.
extra = dict(params.extra_args)
kv = extra.pop("kv_transfer_params", None)
ec = extra.pop("ec_transfer_params", None)
params.extra_args = extra or None
return kv, ec
class ServingRender(BaseServing):
def __init__(
self,
models: OpenAIServingModels | OpenAIModelRegistry,
online_renderer: "OnlineRenderer",
*,
request_logger: RequestLogger | None = None,
tool_server: "ToolServer | None" = None,
) -> None:
super().__init__(
models=models,
model_config=online_renderer.model_config,
request_logger=request_logger,
)
self.online_renderer = online_renderer
self.tool_server = tool_server
self._merge_inline_system = (
AnthropicServingMessages._should_merge_inline_system(online_renderer)
)
self._placeholder_metadata_parser: MultiModalDataParser | None = None
self._placeholder_metadata_parser_failed = False
self.default_sampling_params = (
online_renderer.model_config.get_diff_sampling_param()
)
mc = online_renderer.model_config
self.override_max_tokens = (
self.default_sampling_params.get("max_tokens")
if mc.generation_config not in ("auto", "vllm")
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
)
async def render_chat_request(
self,
request: ChatCompletionRequest,
raw_request: Request | None = None,
) -> GenerateRequest | ErrorResponse:
"""Validate the model and preprocess a chat completion request.
This is the authoritative implementation used directly by the
GPU-less render server and delegated to by OpenAIServingChat.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
logger.error("Error with model %s", error_check_ret)
return error_check_ret
if request.use_beam_search:
return self.create_error_response(
"Beam search is not supported by the render endpoint"
)
result = await self.online_renderer.render_chat(request, skip_mm_cache=True)
if isinstance(result, ErrorResponse):
return result
_, engine_inputs = result
if len(engine_inputs) != 1:
return self.create_error_response(
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
)
engine_input = engine_inputs[0]
prompt_components = extract_prompt_components(self.model_config, engine_input)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
token_ids = list(token_ids)
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_completion_tokens
if request.max_completion_tokens is not None
else request.max_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
truncate_prompt_tokens=request.truncate_prompt_tokens,
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
# Resolve here what OpenAIServingChat passes to the engine:
# `_grammar_from_parser` (set by `adjust_request`) is not serialized.
reasoning_parser_kwargs = self._reasoning_parser_kwargs(request)
parser = None
if reasoning_parser_kwargs is not None:
assert self.online_renderer.parser is not None
parser = self.online_renderer.parser(
self.online_renderer.renderer.get_tokenizer(),
request.tools,
chat_template_kwargs=reasoning_parser_kwargs.chat_template_kwargs,
model_config=self.model_config,
)
reasoning_ended = request.resolve_reasoning_ended(parser, token_ids)
request_id = (
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
)
return GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
model=request.model,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs=reasoning_parser_kwargs,
stream=bool(request.stream),
stream_options=(request.stream_options if request.stream else None),
cache_salt=request.cache_salt,
priority=request.priority,
token_offsets=engine_input.get("prompt_token_offsets"),
)
async def render_messages_request(
self,
request: AnthropicMessagesRequest,
raw_request: Request | None = None,
) -> GenerateRequest | ErrorResponse:
"""Validate the model and preprocess an Anthropic Messages request.
Converts the request to the OpenAI chat format using the same
conversion as the /v1/messages server path, then delegates to
render_chat_request so the rendered tokens match the server exactly.
"""
chat_req = AnthropicServingMessages.to_chat_completion_request(
request, merge_inline_system=self._merge_inline_system
)
chat_req.mm_payload = request.mm_payload
return await self.render_chat_request(chat_req, raw_request)
async def render_completion_request(
self,
request: CompletionRequest,
raw_request: Request | None = None,
) -> list[GenerateRequest] | ErrorResponse:
"""Validate the model and preprocess a completion request.
This is the authoritative implementation used directly by the
GPU-less render server and delegated to by OpenAIServingCompletion.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if request.use_beam_search:
return self.create_error_response(
"Beam search is not supported by the render endpoint"
)
result = await self.online_renderer.render_completion(
request, skip_mm_cache=True
)
if isinstance(result, ErrorResponse):
return result
base_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
generate_requests: list[GenerateRequest] = []
for i, engine_input in enumerate(result):
prompt_components = extract_prompt_components(
self.model_config, engine_input
)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
token_ids = list(token_ids)
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
truncate_prompt_tokens=request.truncate_prompt_tokens,
)
params = request.to_sampling_params(
max_tokens, self.default_sampling_params
)
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
request_id = base_id if len(result) == 1 else f"{base_id}-{i}"
generate_requests.append(
GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
model=request.model,
stream=bool(request.stream),
stream_options=(request.stream_options if request.stream else None),
cache_salt=request.cache_salt,
priority=request.priority,
token_offsets=engine_input.get("prompt_token_offsets"),
)
)
return generate_requests
async def render_responses_request(
self,
request: ResponsesRequest,
) -> GenerateRequest | ErrorResponse:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if request.previous_response_id is not None:
return self.create_error_response(
message=(
"previous_response_id is not supported by the stateless "
"render endpoint."
),
err_type="invalid_request_error",
param="previous_response_id",
)
result = await self.online_renderer.render_responses(
request,
previous_messages=None,
previous_response_outputs=None,
tool_server=self.tool_server,
skip_mm_cache=True,
)
if isinstance(result, ErrorResponse):
return result
engine_input = result.engine_input
prompt_components = extract_prompt_components(self.model_config, engine_input)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_output_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
return GenerateRequest(
request_id=request.request_id,
token_ids=list(token_ids),
features=self._extract_mm_features(
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
model=request.model,
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
stream=bool(request.stream),
cache_salt=request.cache_salt,
priority=request.priority,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
token_offsets=engine_input.get("prompt_token_offsets"),
)
def _reasoning_parser_kwargs(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ReasoningParserKwargs | None:
parser_cls = self.online_renderer.parser
if parser_cls is None or parser_cls.reasoning_parser_cls is None:
return None
return ReasoningParserKwargs(
chat_template_kwargs=self.online_renderer.effective_chat_template_kwargs(
request
)
)
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Fields the EC consumer still requires after embeddings transfer."""
if self._placeholder_metadata_parser_failed:
return set()
if self._placeholder_metadata_parser is None:
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
self._placeholder_metadata_parser = (
MULTIMODAL_REGISTRY.create_processor(
self.model_config
).info.data_parser
)
except Exception:
logger.debug(
"Could not load placeholder metadata fields; "
"mm_metadata will use keep_on_cpu fields only."
)
self._placeholder_metadata_parser_failed = True
return set()
return set(
self._placeholder_metadata_parser.placeholder_metadata_fields(modality)
)
def _extract_mm_features(
self,
engine_input: EngineInput,
*,
mm_payload: MMPayload = "tensors",
) -> MultiModalFeatures | None:
return extract_mm_features(
engine_input,
metadata_fields_for=self._placeholder_metadata_fields,
mm_payload=mm_payload,
)
@@ -108,8 +108,8 @@ def attach_router(app: FastAPI):
app.include_router(router)
if not getattr(app.state.args, "tokens_only", False):
return
# The RLHF dev router registers its own /abort_requests first. Registering
# this one too would only add a shadowed route with a duplicate operation ID.
# An endpoint plugin may already serve /abort_requests. Registering this
# one too would only add a shadowed route with a duplicate operation ID.
has_abort_route = any(
getattr(route, "path", None) == "/abort_requests" for route in app.routes
)
@@ -685,7 +685,6 @@ class DerenderStreamState(BaseModel):
entire accumulated text on each invocation, `parse_delta()` itself is
O(n) yielding a true worst case compute cost of O(n³) per generation.
No caching is performed. Work is bounded by `max_model_len`.
See `OnlineDerenderer._derender_chat_stream_parsed`.
The detokenization strategy carries the incremental decode offsets
directly rather than re-sending the whole token history each chunk.
@@ -823,7 +822,7 @@ class DerenderChatStreamRequest(BaseModel):
`<think>`). `prompt_tokens` is a usage count and cannot serve this
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
Rejected with a 400 when a tool or reasoning
parser is configured and this is omitted. Without it, `parse_delta`
cannot tell whether the prompt left reasoning open and would silently
misclassify reasoning content as plain content. On all paths it also
@@ -1,118 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
from vllm.distributed.elastic_ep.external_elastic_ep import ExternalElasticEPScalePhase
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
logger = init_logger(__name__)
def engine_client(request: Request) -> EngineClient:
return request.app.state.engine_client
router = APIRouter()
@router.post(
"/scale_elastic_ep",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"model": dict},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.REQUEST_TIMEOUT.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def scale_elastic_ep(raw_request: Request):
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(status_code=400, detail="Invalid JSON format") from e
new_data_parallel_size = body.get("new_data_parallel_size")
drain_timeout = body.get("drain_timeout", 120) # Default 2 minutes
if new_data_parallel_size is None:
raise HTTPException(
status_code=400, detail="new_data_parallel_size is required"
)
if not isinstance(new_data_parallel_size, int) or new_data_parallel_size <= 0:
raise HTTPException(
status_code=400,
detail="new_data_parallel_size must be a positive integer",
)
if not isinstance(drain_timeout, int) or drain_timeout <= 0:
raise HTTPException(
status_code=400, detail="drain_timeout must be a positive integer"
)
client = engine_client(raw_request)
try:
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
return JSONResponse(
{
"message": f"Scaled to {new_data_parallel_size} data parallel engines",
}
)
except TimeoutError as e:
raise HTTPException(
status_code=408,
detail="Scale failed due to request drain timeout "
f"after {drain_timeout} seconds",
) from e
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
except Exception as e:
logger.error("Scale failed: %s", e)
raise HTTPException(status_code=500, detail="Scale failed") from e
@router.post("/is_scaling_elastic_ep")
async def is_scaling_elastic_ep(raw_request: Request):
# External operation status comes from the shared store. Middleware gating
# remains process-local, so scaling requests must reach every old-rank API.
try:
status = await engine_client(raw_request).get_external_elastic_ep_status()
except Exception as e:
logger.warning("Failed to query external Elastic EP status: %s", e)
raise HTTPException(
status_code=503,
detail="External Elastic EP status is temporarily unavailable",
) from e
if status is None:
# Non-external EEP modes retain the process-local middleware state.
is_scaling = get_scaling_elastic_ep()
status = {
"phase": (
ExternalElasticEPScalePhase.COMMITTING.value
if is_scaling
else ExternalElasticEPScalePhase.IDLE.value
),
"epoch": None,
"error": None,
"requested_data_parallel_size": None,
}
else:
is_scaling = status["phase"] in (
ExternalElasticEPScalePhase.PREPARING.value,
ExternalElasticEPScalePhase.COMMITTING.value,
)
return JSONResponse({"is_scaling_elastic_ep": is_scaling, **status})
def attach_router(app: FastAPI):
app.include_router(router)
@@ -16,7 +16,3 @@ def register_instrumentator_api_routers(app: FastAPI):
from .metrics import attach_router as metrics_attach_router
metrics_attach_router(app)
from .offline_docs import attach_router as offline_docs_attach_router
offline_docs_attach_router(app)
@@ -1,50 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Offline FastAPI documentation support for air-gapped environments."""
import pathlib
from fastapi import FastAPI
from fastapi.openapi.docs import (
get_swagger_ui_html,
get_swagger_ui_oauth2_redirect_html,
)
from fastapi.staticfiles import StaticFiles
from vllm.logger import init_logger
logger = init_logger(__name__)
def attach_router(app: FastAPI) -> None:
"""Attach offline docs router if enabled via args."""
args = getattr(app.state, "args", None)
if args is None or not getattr(args, "enable_offline_docs", False):
return
static_dir = pathlib.Path(__file__).parent / "static"
if not static_dir.exists():
logger.warning(
"Static directory not found at %s. Offline docs will not be available.",
static_dir,
)
return
app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
@app.get("/docs", include_in_schema=False)
async def custom_swagger_ui_html():
return get_swagger_ui_html(
openapi_url=app.openapi_url,
title=app.title + " - Swagger UI",
oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
swagger_js_url="/static/swagger-ui-bundle.js",
swagger_css_url="/static/swagger-ui.css",
)
@app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False)
async def swagger_ui_redirect():
return get_swagger_ui_oauth2_redirect_html()
logger.info("Offline documentation enabled with vendored static assets")
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -8,7 +8,7 @@ from starlette.datastructures import Headers
from starlette.responses import JSONResponse
from starlette.types import ASGIApp, Receive, Scope, Send
GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
class AuthenticationMiddleware:
@@ -1,4 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SageMaker-specific integration for vLLM."""
@@ -1,156 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import logging
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from typing import Any
import pydantic
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, Response
from vllm.config import ModelConfig
from vllm.entrypoints.generate.factories import get_generate_invocation_types
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.instrumentator.basic import base
from vllm.entrypoints.serve.instrumentator.health import health
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.tasks import SupportedTask
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
# (requires typing_extensions >= 4.13)
RequestType = Any
GetHandlerFn = Callable[[Request], BaseServing | None]
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
"""Snapshot handler levels for loggers that may be affected by third-party
logging configuration side effects (e.g. model_hosting_container_standards
calling configure_root_logger() at import time)."""
loggers = [logging.getLogger(), logging.getLogger("vllm")]
seen: set[int] = set()
snapshot: list[tuple[logging.Handler, int]] = []
for logger in loggers:
for handler in logger.handlers:
if id(handler) not in seen:
seen.add(id(handler))
snapshot.append((handler, handler.level))
return snapshot
def _restore_handler_levels(
snapshot: list[tuple[logging.Handler, int]],
) -> None:
"""Restore handler levels from a snapshot."""
for handler, level in snapshot:
handler.setLevel(level)
def attach_router(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
"""Attach the SageMaker hosting endpoints to the API server.
Handler levels are snapshotted and restored because importing
model_hosting_container_standards may reconfigure root logging.
"""
snapshot = _snapshot_handler_levels()
try:
_attach_router(app, supported_tasks, model_config)
finally:
_restore_handler_levels(snapshot)
def _attach_router(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
"""Register the SageMaker hosting routes (/ping, /invocations) on the
app."""
import model_hosting_container_standards.sagemaker as sagemaker_standards
router = APIRouter()
# NOTE: Construct the TypeAdapters only once
INVOCATION_TYPES = get_generate_invocation_types(supported_tasks, model_config)
INVOCATION_VALIDATORS = [
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
for request_type, (get_handler, endpoint) in INVOCATION_TYPES
]
@router.post("/ping", response_class=Response)
@router.get("/ping", response_class=Response)
@sagemaker_standards.register_ping_handler
async def ping(raw_request: Request) -> Response:
"""Ping check. Endpoint required for SageMaker."""
return await health(raw_request)
@router.post(
"/invocations",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@sagemaker_standards.register_invocation_handler
@sagemaker_standards.stateful_session_manager()
@sagemaker_standards.inject_adapter_id(adapter_path="model")
async def invocations(raw_request: Request):
"""For SageMaker, routes requests based on the request type."""
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=f"JSON decode error: {e}",
) from e
valid_endpoints = [
(validator, endpoint)
for validator, (get_handler, endpoint) in INVOCATION_VALIDATORS
if get_handler(raw_request) is not None
]
for request_validator, endpoint in valid_endpoints:
try:
request = request_validator.validate_python(body)
except pydantic.ValidationError:
continue
return await endpoint(request, raw_request)
type_names = [
t.__name__ if isinstance(t := validator._type, type) else str(t)
for validator, _ in valid_endpoints
]
msg = f"Cannot find suitable handler for request. Expected one of: {type_names}"
res = base(raw_request).create_error_response(message=msg)
return JSONResponse(content=res.model_dump(), status_code=res.error.code)
app.include_router(router)
def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
"""Bootstrap the app with the SageMaker hosting standards.
Handler levels are restored right after the import because importing
model_hosting_container_standards may reconfigure root logging, and
bootstrap must run with the original levels.
"""
snapshot = _snapshot_handler_levels()
try:
import model_hosting_container_standards.sagemaker as sagemaker_standards
app = sagemaker_standards.bootstrap(app)
finally:
_restore_handler_levels(snapshot)
return app
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ASGI middleware for WebSocket Prometheus metrics.
Modeled after prometheus-fastapi-instrumentator, this middleware
transparently instruments WebSocket endpoints with standard metrics
without requiring changes to handler code.
NOTE: This module intentionally has zero vllm imports so that it can
be extracted into a standalone package (similar to
prometheus-fastapi-instrumentator) in the future. Please keep it that way.
"""
import time
from collections.abc import Awaitable
from prometheus_client import Counter, Gauge, Histogram
from starlette.types import ASGIApp, Message, Receive, Scope, Send
# Standard WebSocket metric names (not vllm-specific, following
# the same convention as prometheus-fastapi-instrumentator).
_active_sessions = Gauge(
name="vllm:websocket_connections_active",
documentation="Number of currently active WebSocket connections.",
multiprocess_mode="livesum",
)
_total_sessions = Counter(
name="vllm:websocket_connections_total",
documentation="Total number of WebSocket connections.",
)
_session_duration = Histogram(
name="vllm:websocket_connection_duration_seconds",
documentation="Duration of WebSocket connections in seconds.",
buckets=[0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1800],
)
class WebSocketMetricsMiddleware:
"""Pure ASGI middleware that instruments WebSocket connections.
Tracks active connections (gauge), total connections (counter),
and connection duration (histogram) for all WebSocket endpoints.
Usage::
app.add_middleware(WebSocketMetricsMiddleware)
"""
def __init__(self, app: ASGIApp) -> None:
self.app = app
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
if scope["type"] != "websocket":
return self.app(scope, receive, send)
return self._handle_websocket(scope, receive, send)
async def _handle_websocket(
self, scope: Scope, receive: Receive, send: Send
) -> None:
start_time: float | None = None
async def send_wrapper(message: Message) -> None:
nonlocal start_time
if message["type"] == "websocket.accept":
start_time = time.monotonic()
_active_sessions.inc()
_total_sessions.inc()
await send(message)
try:
await self.app(scope, receive, send_wrapper)
finally:
if start_time is not None:
_active_sessions.dec()
_session_duration.observe(time.monotonic() - start_time)
-151
View File
@@ -49,15 +49,10 @@ if TYPE_CHECKING:
VLLM_TRACE_FUNCTION: int = 0
VLLM_USE_FLASHINFER_SAMPLER: bool = True
VLLM_PP_LAYER_PARTITION: str | None = None
VLLM_CPU_KVCACHE_SPACE: int | None = 0
VLLM_CPU_OMP_THREADS_BIND: str = "auto"
VLLM_CPU_NUM_OF_RESERVED_CPU: int | None = None
VLLM_CPU_CI_ENV: bool = False
VLLM_CPU_ATTN_SPLIT_KV: bool = True
VLLM_ZENTORCH_WEIGHT_PREPACK: bool = True
VLLM_CPU_INT4_W4A8: bool = True
VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
VLLM_XLA_CHECK_RECOMPILATION: bool = False
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: int = 512
VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN: int = 8192
VLLM_XPU_PP_MICROBATCH: bool = True
@@ -97,10 +92,6 @@ if TYPE_CHECKING:
VLLM_USE_PRECOMPILED: bool = False
VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
VLLM_DOCKER_BUILD_CONTEXT: bool = False
VLLM_BUILD_COMMIT: str = "unknown"
VLLM_BUILD_PIPELINE: str = "local"
VLLM_BUILD_URL: str = ""
VLLM_IMAGE_TAG: str = ""
VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
CMAKE_BUILD_TYPE: Literal["Debug", "Release", "RelWithDebInfo"] | None = None
VERBOSE: bool = False
@@ -125,7 +116,6 @@ if TYPE_CHECKING:
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
VLLM_SKIP_P2P_CHECK: bool = False
VLLM_DISABLED_KERNELS: list[str] = []
VLLM_USE_HW_AGNOSTIC: bool = False
VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True
VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda"
VLLM_DISABLE_PYNCCL: bool = False
@@ -142,8 +132,6 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["auto", "asm", "segmented"] = "auto"
VLLM_ROCM_USE_AITER_MHA: bool = True
VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: bool = False
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
@@ -160,12 +148,10 @@ if TYPE_CHECKING:
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
VLLM_DISABLE_COMPILE_CACHE: bool = False
VLLM_REPLICATE_EMBED: bool = False
VLLM_USE_LAYERNAME: bool = True
VLLM_SERVER_DEV_MODE: bool = False
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
VLLM_MLA_DISABLE: bool = False
VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = True
VLLM_RAY_PER_WORKER_GPUS: float = 1.0
VLLM_RAY_BUNDLE_INDICES: str = ""
VLLM_CUDART_SO_PATH: str | None = None
@@ -191,12 +177,10 @@ if TYPE_CHECKING:
VLLM_B12X_MOE_FP4_FORCE_A16: bool = False
VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
VLLM_V1_USE_OUTLINES_CACHE: bool = False
VLLM_TPU_USING_PATHWAYS: bool = False
VLLM_USE_DEEP_GEMM: bool = True
VLLM_MOE_USE_DEEP_GEMM: bool = True
VLLM_USE_DEEP_GEMM_E8M0: bool = True
VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES: bool = True
VLLM_DCP_Q_REPLICATE: bool = False
VLLM_USE_DIRECT_DCP_A2A: bool | None = None
VLLM_USE_DIRECT_DCP_Q_GATHER: bool | None = None
VLLM_USE_DIRECT_DCP_KV_GATHER: bool | None = None
@@ -207,10 +191,6 @@ if TYPE_CHECKING:
] = "relax"
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
VLLM_MOE_SKIP_PADDING: bool = True
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
VLLM_KIMI_K3_AUX_ATTN_RES_STREAM: bool = False
VLLM_KIMI_K3_GEMM_AR: bool = True
VLLM_ENABLE_GEMM_RS: bool = False
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
@@ -224,25 +204,15 @@ if TYPE_CHECKING:
VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False
VLLM_ABLITERATION: str = ""
VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False
VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
VLLM_P2P_SIDE_CHANNEL_HOST: str = "localhost"
VLLM_P2P_SIDE_CHANNEL_PORT: int = 5710
VLLM_P2P_MAX_PEERS: int = 128
VLLM_P2P_HANDSHAKE_TIMEOUT_S: int = 30
VLLM_P2P_IDLE_TIMEOUT_S: int = 300
VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost"
VLLM_EC_SIDE_CHANNEL_PORT: int = 5601
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1
VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9
MOONCAKE_PREFERRED_SEGMENT: str | None = None
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT: bool = True
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
@@ -260,13 +230,10 @@ if TYPE_CHECKING:
VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: int | None = None
VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None
VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480
VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0
VLLM_ENABLE_CUDAGRAPH_GC: bool = False
VLLM_LOOPBACK_IP: str = ""
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
VLLM_ENABLE_COHERE_API: bool = False
VLLM_HAS_FLASHINFER_CUBIN: bool = False
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
@@ -299,9 +266,7 @@ if TYPE_CHECKING:
VLLM_GC_DEBUG: str = ""
VLLM_DEBUG_WORKSPACE: bool = False
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION: bool = False
VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
VLLM_USE_V2_MODEL_RUNNER: bool | None = None
VLLM_LOG_MODEL_INSPECTION: bool = False
@@ -323,11 +288,9 @@ if TYPE_CHECKING:
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True
VLLM_XPU_FORCE_N_CONTIG_WEIGHT: bool = False
VLLM_XPU_USE_SAMPLER_KERNEL: bool = True
VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto"
VLLM_LORA_ENABLE_DUAL_STREAM: bool = False
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
VLLM_NIC_SELECTION_VARS: str = ""
VLLM_ENABLE_HPC_OPS: bool = False
VLLM_REQUEST_BODY_LOG_LEVEL: str = "DEBUG"
@@ -595,9 +558,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Enable batch-invariant mode: deterministic results regardless of
# batch composition. Requires NVIDIA GPU with compute capability >= 9.0.
"VLLM_BATCH_INVARIANT": lambda: bool(int(os.getenv("VLLM_BATCH_INVARIANT", "0"))),
"VLLM_REPLICATE_EMBED": lambda: (
os.getenv("VLLM_REPLICATE_EMBED", "0").strip().lower() in ("1", "true")
),
# Use tensor descriptors for Q/K/V loads and output stores in the
# Triton unified-attention kernel. Enables HW 2D block reads on
# Intel XPU; the non-TD branch is dead-code-eliminated at Triton
@@ -640,10 +600,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
),
# Build provenance metadata embedded in official vllm-openai images.
# Set via Docker ENV at image build time; informational only.
"VLLM_BUILD_COMMIT": lambda: os.environ.get("VLLM_BUILD_COMMIT", "unknown"),
"VLLM_BUILD_PIPELINE": lambda: os.environ.get("VLLM_BUILD_PIPELINE", "local"),
"VLLM_BUILD_URL": lambda: os.environ.get("VLLM_BUILD_URL", ""),
"VLLM_IMAGE_TAG": lambda: os.environ.get("VLLM_IMAGE_TAG", ""),
# CMake build type
# If not set, defaults to "Debug" or "RelWithDebInfo"
# Available options: "Debug", "Release", "RelWithDebInfo"
@@ -839,13 +795,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
),
# Pipeline stage partition strategy
"VLLM_PP_LAYER_PARTITION": lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
# (CPU backend only) CPU key-value cache space.
# default is None and will be set as 4 GB
"VLLM_CPU_KVCACHE_SPACE": lambda: (
int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0"))
if "VLLM_CPU_KVCACHE_SPACE" in os.environ
else None
),
# (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31",
# "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'.
"VLLM_CPU_OMP_THREADS_BIND": lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "auto"),
@@ -856,12 +805,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
if "VLLM_CPU_NUM_OF_RESERVED_CPU" in os.environ
else None
),
# (CPU backend only) whether vLLM is running in a CI environment.
"VLLM_CPU_CI_ENV": lambda: bool(int(os.getenv("VLLM_CPU_CI_ENV", "0"))),
# (CPU backend only) whether to enable attention spilt KV.
"VLLM_CPU_ATTN_SPLIT_KV": lambda: bool(
int(os.getenv("VLLM_CPU_ATTN_SPLIT_KV", "1"))
),
# (Zen CPU backend) eagerly prepack weights into ZenDNN blocked layout
# at model load time. Eliminates per-inference layout conversion overhead.
"VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool(
@@ -1025,16 +968,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"),
# Path to the XLA persistent cache directory.
# Only used for XLA devices such as TPUs.
"VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser(
os.getenv(
"VLLM_XLA_CACHE_PATH",
os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
)
),
# If set, assert on XLA recompilation after each execution step.
"VLLM_XLA_CHECK_RECOMPILATION": lambda: bool(
int(os.getenv("VLLM_XLA_CHECK_RECOMPILATION", "0"))
),
# Enable SPMD mode for TPU backend.
"VLLM_XLA_USE_SPMD": lambda: bool(int(os.getenv("VLLM_XLA_USE_SPMD", "0"))),
# Maximum size (in MB) for logits tensor in sparse MLA indexer prefill chunks.
@@ -1173,9 +1107,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
else os.environ["VLLM_DISABLED_KERNELS"].split(",")
),
# Selects hw-agnostic layers for HF transformer backend
"VLLM_USE_HW_AGNOSTIC": lambda: (
os.getenv("VLLM_USE_HW_AGNOSTIC", "False").lower() in ("true", "1")
),
"VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool(
int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1"))
),
@@ -1277,12 +1208,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# context parallelism: "asm" uses AITER's round-robin ASM decode (gfx950),
# "segmented" the Triton segmented MLA path. "auto" (default) takes "asm"
# wherever it can serve the shape and "segmented" otherwise.
"VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices(
"VLLM_ROCM_AITER_MLA_DCP_VERIFY",
"auto",
["auto", "asm", "segmented"],
case_sensitive=False,
),
# Small-head (<16) AITER MLA decode kernel selection. Small head counts
# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
# the Gluon small-head kernel or through the padded persistent-scheduling
@@ -1291,12 +1216,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# PS ASM decode; "gluon" forces the Gluon path wherever a build exists;
# "asm" forces the padded PS ASM decode. On gfx942/CDNA3 there is no Gluon
# build, so the ASM path is always used regardless of this setting.
"VLLM_ROCM_AITER_MLA_ASM_PADDING": env_with_choices(
"VLLM_ROCM_AITER_MLA_ASM_PADDING",
"auto",
["auto", "gluon", "asm"],
case_sensitive=False,
),
# Whether to use aiter mha ops.
# By default is enabled.
"VLLM_ROCM_USE_AITER_MHA": lambda: (
@@ -1439,9 +1358,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Deprecated and ignored: RayExecutorV2 is the only Ray executor. Kept so
# out-of-tree platforms that still read it keep working; will be removed
# in v0.34.
"VLLM_USE_RAY_V2_EXECUTOR_BACKEND": lambda: bool(
int(os.getenv("VLLM_USE_RAY_V2_EXECUTOR_BACKEND", "1"))
),
# In some system, find_loaded_library() may not work. So we allow users to
# specify the path through environment variable VLLM_CUDART_SO_PATH.
"VLLM_CUDART_SO_PATH": lambda: os.getenv("VLLM_CUDART_SO_PATH", None),
@@ -1546,9 +1462,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
os.environ.get("VLLM_V1_USE_OUTLINES_CACHE", "0") == "1"
),
# Whether using Pathways
"VLLM_TPU_USING_PATHWAYS": lambda: bool(
"proxy" in os.getenv("JAX_PLATFORMS", "").lower()
),
# Allow use of DeepGemm kernels for fused moe ops.
"VLLM_USE_DEEP_GEMM": lambda: bool(int(os.getenv("VLLM_USE_DEEP_GEMM", "1"))),
# Allow use of DeepGemm specifically for MoE fused ops (overrides only MoE).
@@ -1564,7 +1477,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
int(os.getenv("VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES", "1"))
),
# Opt-in MLA DCP query replication: skip the decode query all-gather.
"VLLM_DCP_Q_REPLICATE": lambda: bool(int(os.getenv("VLLM_DCP_Q_REPLICATE", "0"))),
# DeepGemm JITs the kernels on-demand. The warmup attempts to make DeepGemm
# JIT all the required kernels before model execution so there is no
# JIT'ing in the hot-path. However, this warmup increases the engine
@@ -1602,24 +1514,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
# per layer, so it only wins at low token counts: intended for decode
# instances in a P/D disaggregated deployment, not for prefill or unified
# serving.
"VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool(
int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0"))
),
# Kimi K3 only, and unrelated to the MoE flags above. Tap the pre-norm
# AttnRes mixture, rather than the post-mixture sum, as the auxiliary
# hidden state handed to a DFlash drafter. This changes the numerics the
# speculator sees, so it is off by default while the effect is measured.
"VLLM_KIMI_K3_AUX_ATTN_RES_STREAM": lambda: bool(
int(os.getenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "0"))
),
# Use the SM100 BF16 GEMM-AR kernel for eligible Kimi-K3 row-parallel
# attention projections. All TP ranks must belong to one NVLink domain.
"VLLM_KIMI_K3_GEMM_AR": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_AR", "1"))),
# Fuse eligible sequence-parallel row-parallel projections with their TP
# reduce-scatter using the SM100 BF16/MXFP8 GEMM-RS kernel (Kimi-K3
# attention/shared-expert projections, DeepSeek-V4.1 ``wo_b``). All TP
# ranks must belong to one NVLink domain.
"VLLM_ENABLE_GEMM_RS": lambda: bool(int(os.getenv("VLLM_ENABLE_GEMM_RS", "0"))),
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
@@ -1666,13 +1570,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
int(os.getenv("VLLM_DISABLE_REQUEST_ID_RANDOMIZATION", "0"))
),
# IP address used for NIXL handshake between remote agents.
"VLLM_NIXL_SIDE_CHANNEL_HOST": lambda: os.getenv(
"VLLM_NIXL_SIDE_CHANNEL_HOST", "localhost"
),
# Port used for NIXL handshake between remote agents.
"VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(
os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600")
),
# Address the P2P KV-offload control socket binds to. Defaults to
# ``localhost`` (loopback only); must be set to the node IP for
# cross-host P2P so remote peers can reach the socket.
@@ -1700,39 +1598,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_P2P_IDLE_TIMEOUT_S": lambda: int(os.getenv("VLLM_P2P_IDLE_TIMEOUT_S", "300")),
# IP address used for the EC connector's ZMQ side channel
# (producer ROUTER bind, consumer DEALER dial).
"VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv(
"VLLM_EC_SIDE_CHANNEL_HOST", "localhost"
),
# Port for the EC connector's ZMQ side channel; advertised to peers
# via `ec_transfer_params.peer_port` on the producer's response.
"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
),
# Per-request timeout (seconds) when a prefiller worker registers with
# the Mooncake bootstrap server.
"VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float(
os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0")
),
# Port used for Mooncake handshake between remote agents.
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
),
# Log per-batch memory/disk tier breakdown on external GETs.
"VLLM_MOONCAKE_STORE_TIER_LOG": lambda: (
os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1")
),
# Number of parallel KV-load receive threads per worker rank. Lets the
# per-request control overhead (Python prep + master key lookup) of one
# request overlap with the RDMA transfer of another, keeping the transfer
# engine's queue pairs busy. Helps when that overhead is significant or
# per-request batches are too small to saturate the link on their own.
"VLLM_MOONCAKE_LOAD_RECV_THREADS": lambda: int(
os.getenv("VLLM_MOONCAKE_LOAD_RECV_THREADS", "1")
),
# Fraction of the owner's DirectIO staging buffer to fill per GET batch.
"VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float(
os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9")
),
# Pin this rank to a specific owner segment ("host:port").
"MOONCAKE_PREFERRED_SEGMENT": lambda: os.getenv("MOONCAKE_PREFERRED_SEGMENT"),
# Override the hostname the rank registers as a Mooncake requester.
@@ -1788,18 +1665,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MOE_ROUTING_SIMULATION_STRATEGY", ""
).lower(),
# Regex timeout for use by the vLLM tool parsing plugins.
"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int(
os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")
),
# Enforce function parameter schemas in structural-tag based tool calling.
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: (
os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1")
),
# Constrain every `beam` output to its response template's format, also
# without tool or structured output constraints.
"VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT": lambda: (
os.getenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "True").lower() in ("true", "1")
),
# Control the max chunk bytes (in MB) for the rpc message queue.
# Object larger than this threshold will be broadcast to worker
# processes via zmq.
@@ -1849,9 +1720,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
),
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
),
# If set, it means we pre-downloaded cubin files and flashinfer will
# read the cubin files directly.
"VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
@@ -1891,12 +1759,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
),
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``
# and its render endpoint at ``POST /cohere/v2/chat/render``.
# Default off
"VLLM_ENABLE_COHERE_API": lambda: bool(
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
),
# If set, use the fp8 mfma in rocm paged attention.
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
@@ -2056,9 +1918,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Emergency rollback for the DeepSeek-V4 NVIDIA MegaMoE path. By default,
# DeepGEMM computes replicated FP8 shared experts in the same persistent
# SM100 kernel as the routed FP4 experts.
"VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION": lambda: bool(
int(os.getenv("VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION", "0"))
),
# Limits when we run shared_experts in a separate stream.
# We found out that for large batch sizes, the separate stream
# execution is not beneficial (most likely because of the input clone)
@@ -2074,9 +1933,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# and the cross-stream sync becomes pure overhead. Set to 0 to disable
# the multi-stream path entirely. See #PR 41526 for the empirical result
# for the default value of 1024 tokens.
"VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int(
os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024")
),
# Format for saving torch.compile cache artifacts
# - "binary": saves as binary file
# Safe for multiple vllm serve processes accessing the same torch compile cache.
@@ -2175,9 +2031,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# activations to per-token int8 (int4_gemm_w4a8) for large token counts.
# The two oneDNN backends are int4-only; ARK also serves int2.
# Which one is fastest is device-dependent, so this is left as an opt-in.
"VLLM_XPU_INC_WNA16_BACKEND": env_with_choices(
"VLLM_XPU_INC_WNA16_BACKEND", "auto", ["auto", "ark", "w4a16", "w4a8"]
),
# Direct DCP ops default on when applicable; set to 1 to enforce or 0 to disable.
"VLLM_USE_DIRECT_DCP_A2A": lambda: maybe_convert_bool(
os.getenv("VLLM_USE_DIRECT_DCP_A2A")
@@ -2212,7 +2065,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# fused into gated_mla_gemm; elementwise gating only.
# Each op additionally checks its own shape / dtype constraints and falls
# back to the eager path when they do not hold.
"VLLM_ENABLE_HPC_OPS": lambda: bool(int(os.getenv("VLLM_ENABLE_HPC_OPS", "0"))),
# Whether to skip version suffix when building package
"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
@@ -2308,13 +2160,11 @@ def compile_factors() -> dict[str, object]:
# HOME or the XDG roots silently invalidates every compile cache
# (VLLM_CACHE_ROOT above and VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR below
# are already ignored for the same reason).
"VLLM_XLA_CACHE_PATH",
"VLLM_CONFIG_ROOT",
"LD_LIBRARY_PATH",
"VLLM_SERVER_DEV_MODE",
"VLLM_DP_MASTER_IP",
"VLLM_DP_MASTER_PORT",
"VLLM_NIXL_SIDE_CHANNEL_HOST",
"VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
"VLLM_MODEL_REDIRECT_PATH",
"VLLM_HOST_IP",
@@ -2369,7 +2219,6 @@ def compile_factors() -> dict[str, object]:
"VLLM_WORKER_MULTIPROC_METHOD",
"VLLM_ENABLE_V1_MULTIPROCESSING",
"VLLM_V1_OUTPUT_PROC_CHUNK_SIZE",
"VLLM_CPU_KVCACHE_SPACE",
"VLLM_CPU_MOE_PREPACK",
"VLLM_ZENTORCH_WEIGHT_PREPACK",
"VLLM_TEST_FORCE_LOAD_FORMAT",
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,596 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utilities for Punica kernel construction."""
from vllm.triton_utils import tl, triton
@triton.jit
def _accumulate_mm(
tiled_a,
tiled_b,
accumulator,
a_scale_ptr,
b_scale_ptr,
a_scale_k_stride,
b_scale_k_stride,
iter_k,
group_k: tl.constexpr,
group_n: tl.constexpr,
use_fp8_w8a8: tl.constexpr,
):
"""Core matrix multiplication and accumulation logic with quantization support.
Args:
tiled_a (tl.tensor): Loaded tile from A matrix
tiled_b (tl.tensor): Loaded tile from B matrix
accumulator (tl.tensor): Current accumulator value
a_scale_ptr (tl.tensor): Scale pointer for A matrix
b_scale_ptr (tl.tensor): Scale pointer for B matrix
a_scale_k_stride (int): K dimension stride for A's block-wise scales
b_scale_k_stride (int): K dimension stride for B's block-wise scales
iter_k (int): Current iteration's global K offset
group_k: Block size for K dimension in block-wise quantization
group_n: Block size for N dimension in block-wise quantization
use_fp8_w8a8: Whether using FP8 W8A8 quantization
"""
if use_fp8_w8a8:
if group_k > 0 and group_n > 0:
# Block-wise quantization: scales are loaded per block
offs_ks = iter_k // group_k
# a_scale_ptr is (BLOCK_M,) tensor of base pointers per row
# Load scale for current K-group, result shape: (BLOCK_M,)
a_scale = tl.load(a_scale_ptr + offs_ks * a_scale_k_stride)
# b_scale_ptr is (BLOCK_N,) tensor with N-offset pre-baked
# Load scale for current K-group, result shape: (BLOCK_N,)
b_scale = tl.load(b_scale_ptr + offs_ks * b_scale_k_stride)
accumulator += (
tl.dot(tiled_a, tiled_b) * a_scale[:, None] * b_scale[None, :]
)
else:
# Tensor-wise or per-channel: accumulate and scale at end
accumulator = tl.dot(tiled_a, tiled_b, acc=accumulator)
else:
accumulator += tl.dot(tiled_a, tiled_b)
return accumulator
@triton.jit
def fp8_mm_k(
a_ptr,
b_ptr,
a_scale_ptr,
b_scale_ptr,
ak_stride,
bk_stride,
a_scale_k_stride,
b_scale_k_stride,
offset_k,
K: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
EVEN_K: tl.constexpr,
SPLIT_K: tl.constexpr,
group_k: tl.constexpr,
group_n: tl.constexpr,
use_fp8_w8a8: tl.constexpr,
per_channel_quant: tl.constexpr,
CAST_TYPE: tl.constexpr,
b_dtype: tl.constexpr,
USE_GDC: tl.constexpr,
base_k,
):
"""FP8-compatible matrix multiplication kernel with quantization support.
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
B (k x n), iterate through the K dimension to compute the partial/complete
matrix block product with proper dequantization.
Args:
a_ptr (tl.tensor): Array of pointers, identifying rows of A
(FP8 or other dtype)
b_ptr (tl.tensor): Array of pointers, identifying columns of B
(FP8 dtype)
a_scale_ptr (tl.tensor): Scale pointer for A matrix
(per-token or block-wise)
b_scale_ptr (tl.tensor): Scale pointer for B matrix
(per-channel or block-wise)
ak_stride (int): K dimension stride of the A matrix
bk_stride (int): K dimension stride of the B matrix
a_scale_k_stride (int): K dimension stride for A's block-wise scales
b_scale_k_stride (int): K dimension stride for B's block-wise scales
offset_k (int): Base offset along K dimension
K: Length of the K dimension
BLOCK_M: M dimension of the output block m x n
BLOCK_N: N dimension of the output block m x n
BLOCK_K: K dimension atom
EVEN_K: True if the blocks of A and B can be loaded without masking
SPLIT_K: Parameter signifying parallelism in the K dimension
group_k: Block size for K dimension in block-wise quantization
group_n: Block size for N dimension in block-wise quantization
use_fp8_w8a8: Whether using FP8 W8A8 quantization
per_channel_quant: Whether using per-channel quantization
CAST_TYPE: if True, cast the values from the A matrix to the B
matrix dtype.
b_dtype: datatype of the B matrix
USE_GDC: Whether to use PDL. True indicates use.
base_k (int): Base offset along K dimension for current SPLIT_K group
"""
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
# Step size along K for each iteration
STEP_K = BLOCK_K * SPLIT_K
# Total number of iterations (compile-time constant)
num_iters = tl.cdiv(K, STEP_K)
for k in range(num_iters):
# Current iteration's global K offset
iter_k = k * STEP_K + base_k
block_end = iter_k + BLOCK_K
# Skip iterations that are entirely past the K boundary
if not EVEN_K and iter_k >= K:
pass
elif EVEN_K or block_end <= K:
# No masking needed: either K is evenly divisible (EVEN_K)
# or this block fits entirely within K
tiled_b = tl.load(b_ptr)
if USE_GDC:
tl.extra.cuda.gdc_wait()
tiled_a = tl.load(a_ptr)
if CAST_TYPE:
tiled_a = tiled_a.to(b_dtype)
accumulator = _accumulate_mm(
tiled_a,
tiled_b,
accumulator,
a_scale_ptr,
b_scale_ptr,
a_scale_k_stride,
b_scale_k_stride,
iter_k,
group_k,
group_n,
use_fp8_w8a8,
)
else:
# Partial block at the tail: mask out-of-bounds elements
k_offsets = tl.arange(0, BLOCK_K)
mask = iter_k + k_offsets < K
tiled_b = tl.load(b_ptr, mask=mask[:, None], other=0.0)
if USE_GDC:
tl.extra.cuda.gdc_wait()
tiled_a = tl.load(a_ptr, mask=mask[None, :], other=0.0)
if CAST_TYPE:
tiled_a = tiled_a.to(b_dtype)
accumulator = _accumulate_mm(
tiled_a,
tiled_b,
accumulator,
a_scale_ptr,
b_scale_ptr,
a_scale_k_stride,
b_scale_k_stride,
iter_k,
group_k,
group_n,
use_fp8_w8a8,
)
a_ptr += STEP_K * ak_stride
b_ptr += STEP_K * bk_stride
return accumulator
@triton.jit
def do_shrink_kernel_fp8(
pid_n,
pid_sk,
slice_id,
lora_index,
input_ptr,
lora_ptr,
out_ptr,
a_scale_ptr,
b_scale_ptr,
N,
K,
M_LEN,
ram,
# input strides
input_d0_stride,
input_d1_stride,
# lora strides
lora_d0_stride,
lora_d1_stride,
lora_d2_stride,
# scale strides
a_scale_m_stride,
a_scale_k_stride,
b_scale_l_stride,
b_scale_n_stride,
b_scale_k_stride,
# output strides
output_d0_stride,
output_d1_stride,
output_d2_stride,
scaling,
# block size for block-wise quantization
group_n: tl.constexpr,
group_k: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
EVEN_K: tl.constexpr,
SPLIT_K: tl.constexpr,
SLICE_NUM: tl.constexpr,
USE_GDC: tl.constexpr,
use_fp8_w8a8: tl.constexpr,
per_channel_quant: tl.constexpr,
launch_pdl: tl.constexpr,
):
"""Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice, compute the
matrix product and store in the appropriate output location.
"""
# Identify the lora_ptr from slice_id.
if SLICE_NUM == 1:
cur_lora_ptr = lora_ptr
cur_b_scale_ptr = b_scale_ptr
else:
cur_lora_ptr = (
tl.load(lora_ptr + slice_id).to(tl.pointer_type(tl.float8e4nv))
if b_scale_ptr is not None
else tl.load(lora_ptr + slice_id).to(
tl.pointer_type(input_ptr.dtype.element_ty)
)
)
cur_b_scale_ptr = (
tl.load(b_scale_ptr + slice_id).to(tl.pointer_type(tl.float32))
if b_scale_ptr is not None
else b_scale_ptr
)
# Identify the column indices of B to process.
offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
# Identify A and B block pointers
offset_k = pid_sk * BLOCK_K + tl.arange(0, BLOCK_K)
a_ptr = (
input_ptr + ram[:, None] * input_d0_stride + offset_k[None, :] * input_d1_stride
)
b_ptr = (
cur_lora_ptr
+ lora_d0_stride * lora_index
+ rbn[None, :] * lora_d1_stride
+ offset_k[:, None] * lora_d2_stride
)
# Load scales for tensor-wise or per-channel quantization (outside the loop)
# Block-wise scales are loaded inside fp8_mm_k
if use_fp8_w8a8:
if group_k > 0 and group_n > 0:
# Block-wise: compute scale pointers for fp8_mm_k
# a_scale: per-row base pointers, shape (BLOCK_M,)
# Each pointer points to the start of that row's scale data
mm_a_scale_ptr = a_scale_ptr + ram * a_scale_m_stride
# b_scale: pre-compute N-dimension offset
# We need to bake in the N-group offset since fp8_mm_k doesn't know pid_n
n_offset = pid_n * BLOCK_N
offs_ns = (n_offset + tl.arange(0, BLOCK_N)) // group_n
# Base pointer with lora offset + N-group offset baked in, shape (BLOCK_N,)
mm_b_scale_ptr = (
cur_b_scale_ptr
+ lora_index * b_scale_l_stride
+ offs_ns * b_scale_n_stride
)
elif per_channel_quant:
# Per-channel for weights, per-token for activations
b_scale_ptrs = (
cur_b_scale_ptr + lora_index * b_scale_l_stride + rbn * b_scale_n_stride
)
b_scale = tl.load(b_scale_ptrs)
# Per-token activation scale
a_scale = tl.load(a_scale_ptr + ram * a_scale_m_stride)[:, None]
# For non-block-wise, pass original pointers (not used in mm loop)
mm_a_scale_ptr = a_scale_ptr
mm_b_scale_ptr = cur_b_scale_ptr
else:
# Tensor-wise quantization
a_scale = tl.load(a_scale_ptr) if a_scale_ptr is not None else 1.0
b_scale = tl.load(cur_b_scale_ptr + lora_index * b_scale_l_stride)
# For non-block-wise, pass original pointers (not used in mm loop)
mm_a_scale_ptr = a_scale_ptr
mm_b_scale_ptr = cur_b_scale_ptr
else:
# Non-quantized path
mm_a_scale_ptr = a_scale_ptr
mm_b_scale_ptr = cur_b_scale_ptr
# Compute partial/complete block matrix product.
accumulator = fp8_mm_k(
a_ptr,
b_ptr,
mm_a_scale_ptr,
mm_b_scale_ptr,
input_d1_stride,
lora_d2_stride,
a_scale_k_stride,
b_scale_k_stride,
offset_k,
K,
BLOCK_M,
BLOCK_N,
BLOCK_K,
EVEN_K,
SPLIT_K,
group_k,
group_n,
use_fp8_w8a8,
per_channel_quant,
False,
cur_lora_ptr.dtype.element_ty,
USE_GDC,
base_k=pid_sk * BLOCK_K,
)
# GDC launch dependents hints the runtime system to launch dependent kernels.
if USE_GDC:
tl.extra.cuda.gdc_launch_dependents()
# Apply dequantization scales for tensor-wise/per-channel quantization
if use_fp8_w8a8:
if group_k > 0 and group_n > 0:
# Block-wise: already applied in fp8_mm_k
pass
else:
# Tensor-wise or per-channel: apply scales after accumulation
accumulator = accumulator * a_scale * b_scale
# Apply LoRA scaling factor
accumulator *= scaling
# Identify the C output pointers to store the results of the accumulator.
offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
offset_cm = tl.arange(0, BLOCK_M)
cur_out_ptr = out_ptr if SLICE_NUM == 1 else out_ptr + slice_id * output_d0_stride
c_ptr = (
cur_out_ptr
+ ram[:, None] * output_d1_stride
+ offset_cn[None, :] * output_d2_stride
)
c_mask = (offset_cm[:, None] < M_LEN) & (offset_cn[None, :] < N)
# Cast accumulator to output dtype
accumulator = accumulator.to(out_ptr.dtype.element_ty)
# handles write-back with reduction-splitting
if SPLIT_K == 1:
tl.store(c_ptr, accumulator, mask=c_mask)
else:
tl.atomic_add(c_ptr, accumulator, mask=c_mask, sem="relaxed")
@triton.jit
def do_expand_kernel_fp8(
pid_n,
lora_index,
slice_id,
input_ptr,
lora_ptr,
out_ptr,
a_scale_ptr,
b_scale_ptr,
N,
K,
M_LEN,
ram, # array identifying the rows of Input ptr to operate on
slice_start_loc,
# input ptr strides
input_d0_stride,
input_d1_stride,
input_d2_stride,
# lora ptr strides
ls_d0_ptr,
ls_d1_ptr,
ls_d2_ptr,
# scale strides
a_scale_m_stride,
a_scale_k_stride,
b_scale_l_stride,
b_scale_n_stride,
b_scale_k_stride,
# out ptr strides
output_d0_stride,
output_d1_stride,
# block size for block-wise quantization
group_n: tl.constexpr,
group_k: tl.constexpr,
# constants
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
SAME_STRIDE: tl.constexpr,
SLICE_NUM: tl.constexpr,
EVEN_K: tl.constexpr,
CAST_TYPE: tl.constexpr,
ADD_INPUTS: tl.constexpr,
USE_GDC: tl.constexpr,
use_fp8_w8a8: tl.constexpr,
per_channel_quant: tl.constexpr,
):
"""FP8-compatible expand kernel for LoRA.
Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice,
compute the matrix product with FP8 quantization support and store in
the appropriate output location.
For expand kernel, the input (shrink output) may be in FP32/FP16/BF16,
while the LoRA B weights can be in FP8.
Supports:
- FP8 W8A8 quantization for LoRA B weights
- Block-wise quantization with configurable group_k and group_n
- Per-channel quantization
- Tensor-wise quantization
"""
# ls_d*_ptr can be either an integer or a pointer
if SAME_STRIDE:
cur_lora_d0_stride = ls_d0_ptr
cur_lora_d1_stride = ls_d1_ptr
cur_lora_d2_stride = ls_d2_ptr
else:
cur_lora_d0_stride = tl.load(ls_d0_ptr + slice_id)
cur_lora_d1_stride = tl.load(ls_d1_ptr + slice_id)
cur_lora_d2_stride = tl.load(ls_d2_ptr + slice_id)
# Identify the input_ptr and lora_ptr from slice_id.
if SLICE_NUM == 1:
cur_input_ptr = input_ptr
if use_fp8_w8a8:
cur_lora_ptr = lora_ptr
cur_b_scale_ptr = b_scale_ptr
else:
cur_lora_ptr = lora_ptr
cur_b_scale_ptr = b_scale_ptr # May be None for non-quantized
else:
cur_input_ptr = input_ptr + slice_id * input_d0_stride
if use_fp8_w8a8:
cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
tl.pointer_type(tl.float8e4nv)
)
cur_b_scale_ptr = tl.load(b_scale_ptr + slice_id).to(
tl.pointer_type(tl.float32)
)
else:
cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
tl.pointer_type(out_ptr.dtype.element_ty)
)
cur_b_scale_ptr = (
tl.load(b_scale_ptr + slice_id).to(tl.pointer_type(tl.float32))
if b_scale_ptr is not None
else None
)
# Identify the column indices of B to process.
offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
# Identify A and B block pointers
offset_k = tl.arange(0, BLOCK_K)
a_ptr = (
cur_input_ptr
+ ram[:, None] * input_d1_stride
+ offset_k[None, :] * input_d2_stride
)
b_ptr = (
cur_lora_ptr
+ cur_lora_d0_stride * lora_index
+ offset_k[:, None] * cur_lora_d2_stride
+ rbn[None, :] * cur_lora_d1_stride
)
# Setup scale pointers for FP8/INT8 quantization
if use_fp8_w8a8:
if group_k > 0 and group_n > 0:
# Block-wise quantization - compute scale pointers for fp8_mm_k
# a_scale: per-row base pointers, shape (BLOCK_M,)
mm_a_scale_ptr = a_scale_ptr + ram * a_scale_m_stride
# b_scale: pre-compute N-dimension offset since fp8_mm_k doesn't know pid_n
n_offset = pid_n * BLOCK_N
offs_ns = (n_offset + tl.arange(0, BLOCK_N)) // group_n
# Base pointer with lora offset + N-group offset baked in, shape (BLOCK_N,)
mm_b_scale_ptr = (
cur_b_scale_ptr
+ lora_index * b_scale_l_stride
+ offs_ns * b_scale_n_stride
)
elif per_channel_quant:
# Per-channel for weights, shape (BLOCK_N,)
b_scale_ptrs = (
cur_b_scale_ptr + lora_index * b_scale_l_stride + rbn * b_scale_n_stride
)
b_scale = tl.load(b_scale_ptrs)
# Per-token activation scale, only if a_scale_ptr provided
a_scale = tl.load(a_scale_ptr + ram * a_scale_m_stride)[:, None]
# For non-block-wise, pass original pointers (not used in mm loop)
mm_a_scale_ptr = a_scale_ptr
mm_b_scale_ptr = cur_b_scale_ptr
else:
# Tensor-wise quantization
a_scale = tl.load(a_scale_ptr) if a_scale_ptr is not None else 1.0
b_scale = tl.load(cur_b_scale_ptr + lora_index * b_scale_l_stride)
# For non-block-wise, pass original pointers (not used in mm loop)
mm_a_scale_ptr = a_scale_ptr
mm_b_scale_ptr = cur_b_scale_ptr
else:
# Non-quantized path
mm_a_scale_ptr = a_scale_ptr
mm_b_scale_ptr = cur_b_scale_ptr
# Compute the block matrix product using fp8_mm_k
# Note: For expand kernel, SPLIT_K=1, so we pass 1 for SPLIT_K
accumulator = fp8_mm_k(
a_ptr,
b_ptr,
mm_a_scale_ptr,
mm_b_scale_ptr,
input_d2_stride, # ak_stride
cur_lora_d2_stride, # bk_stride
a_scale_k_stride,
b_scale_k_stride,
offset_k,
K,
BLOCK_M,
BLOCK_N,
BLOCK_K,
EVEN_K,
1, # SPLIT_K = 1 for expand kernel
group_k,
group_n,
use_fp8_w8a8,
per_channel_quant,
CAST_TYPE, # CAST_TYPE - cast FP8 B to A's dtype
cur_lora_ptr.dtype.element_ty,
USE_GDC,
base_k=0,
)
# Apply dequantization scales for non-block-wise quantization
if use_fp8_w8a8:
if group_k > 0 and group_n > 0:
pass # Already applied per block in fp8_mm_k
else:
# Tensor-wise or per-channel: apply scales after accumulation
accumulator = accumulator * a_scale * b_scale
tiled_c = accumulator.to(out_ptr.dtype.element_ty)
if SLICE_NUM == 1:
cur_slice_start = slice_start_loc
else:
cur_slice_start = tl.load(slice_start_loc + slice_id)
# Identify the C output pointers to store the results of the accumulator.
offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + cur_slice_start
offset_cm = tl.arange(0, BLOCK_M)
c_ptr = (
out_ptr
+ ram[:, None] * output_d0_stride
+ offset_cn[None, :] * output_d1_stride
)
c_mask = (offset_cm[:, None] < M_LEN) & (offset_cn[None, :] < (cur_slice_start + N))
if ADD_INPUTS:
tiled_out = tl.load(c_ptr, mask=c_mask)
tiled_c += tiled_out
tl.store(c_ptr, tiled_c, mask=c_mask)
@@ -1,146 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"24": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 5
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 2
},
"48": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"96": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"1536": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"3072": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
}
}
@@ -1,146 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"24": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 5
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"48": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 5
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"96": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 5
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 4
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
}
}
@@ -1,218 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"24": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"48": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"96": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"1536": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"3072": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"5120": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"9216": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"13312": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"17408": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"25600": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"33792": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"41984": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"50176": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"58368": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
}
}
@@ -1,218 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 5
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"24": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"48": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"96": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"5120": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"9216": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"13312": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"17408": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"25600": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"33792": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"41984": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"50176": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"58368": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
}
}
@@ -1,146 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"24": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"48": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 4
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"96": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"128": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
}
}
@@ -1,218 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"24": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"48": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"96": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"1536": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"3072": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"5120": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"9216": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"13312": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"17408": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"25600": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"33792": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"41984": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"50176": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"58368": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
}
}
@@ -1,146 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"24": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"48": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"64": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"96": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"128": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
}
}
@@ -1,218 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"24": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"48": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"64": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"96": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"128": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"5120": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 4
},
"9216": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3
},
"13312": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3
},
"17408": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3
},
"25600": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 4
},
"33792": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3
},
"41984": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3
},
"50176": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3
},
"58368": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3
}
}
@@ -1,146 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"24": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"48": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"96": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
}
}
@@ -1,218 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"24": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"48": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"96": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 256,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"1536": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"3072": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"5120": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"9216": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"13312": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"17408": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"25600": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"33792": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"41984": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"50176": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"58368": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
}
}
@@ -1,218 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 5
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 5
},
"24": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"48": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"96": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 4
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 4
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"5120": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"9216": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"13312": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"17408": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"25600": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"33792": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"41984": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"50176": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"58368": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
}
}
@@ -1,146 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"24": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"32": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"48": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"64": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"96": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"128": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
}
}
@@ -1,218 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"24": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"48": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"96": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"1536": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"3072": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"5120": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"9216": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"13312": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"17408": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"25600": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"33792": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"41984": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"50176": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
},
"58368": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4
}
}
@@ -1,218 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 5
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"24": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 5
},
"48": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"96": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 64,
"num_warps": 4,
"num_stages": 4
},
"256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 4
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"5120": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"9216": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"13312": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"17408": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"25600": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"33792": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"41984": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"50176": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"58368": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
}
}
@@ -1,146 +0,0 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 256,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"24": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"32": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"48": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"64": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"96": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4
},
"128": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"1536": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3
}
}

Some files were not shown because too many files have changed in this diff Show More