[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -339,17 +339,6 @@ def fused_add_rms_norm(
|
||||
torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
|
||||
|
||||
|
||||
def muse_spark_mul_tanh_soft_cap(
|
||||
x: torch.Tensor,
|
||||
multiplier: float,
|
||||
soft_cap_temp: float,
|
||||
) -> torch.Tensor:
|
||||
"""Cast to FP32 and apply ``temp * tanh(x * multiplier / temp)`` in one pass."""
|
||||
out = torch.empty_like(x, dtype=torch.float32)
|
||||
torch.ops._C.muse_spark_mul_tanh_soft_cap(out, x, multiplier, soft_cap_temp)
|
||||
return out
|
||||
|
||||
|
||||
def fused_qk_norm_rope(
|
||||
qkv: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
@@ -2609,197 +2598,6 @@ def reshape_and_cache_flash(
|
||||
)
|
||||
|
||||
|
||||
def fused_qknorm_rope_cache(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rotary_dim: int,
|
||||
eps: float,
|
||||
norm_weight_bias: float = 0.0,
|
||||
kv_cache_dtype: str = "auto",
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fuse per-head RMSNorm, partial NeoX RoPE and KV insertion.
|
||||
|
||||
Norm weights may be FP32 independently of QKV. A zero rotary dimension
|
||||
skips RoPE; norm outputs are rounded to QKV dtype before rotation.
|
||||
"""
|
||||
if rotary_dim < 0 or rotary_dim > head_dim or rotary_dim % 8:
|
||||
raise ValueError("rotary_dim must be a multiple of 8 in [0, head_dim]")
|
||||
q = qkv.new_empty((qkv.shape[0], num_q_heads * head_dim))
|
||||
k = qkv.new_empty((qkv.shape[0], num_kv_heads * head_dim))
|
||||
v = torch.empty_like(k)
|
||||
cos, sin = cos_sin_cache.chunk(2, dim=-1)
|
||||
torch.ops._C.fused_qknorm_rope_cache(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos,
|
||||
sin,
|
||||
positions,
|
||||
slot_mapping,
|
||||
key_cache,
|
||||
value_cache,
|
||||
head_dim,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
rotary_dim // 2,
|
||||
eps,
|
||||
norm_weight_bias,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
return q, k, v
|
||||
|
||||
|
||||
def fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
cos_sin_cache: torch.Tensor | None = None,
|
||||
rotary_dim: int = 0,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
kv_cache: torch.Tensor | None = None,
|
||||
block_size: int = 0,
|
||||
kv_cache_dtype: str = "auto",
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
q_fp8_out: torch.Tensor | None = None,
|
||||
q_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""In-place weightless QK RMSNorm + GPT-J RoPE, optionally writing k/v to cache.
|
||||
|
||||
When ``kv_cache`` is given, the caller must skip the attention layer's own KV write.
|
||||
With ``q_fp8_out``, write scaled E4M3 Q there instead of rewriting Q in ``qkv``.
|
||||
"""
|
||||
torch.ops._C.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
cos_sin_cache,
|
||||
rotary_dim,
|
||||
slot_mapping,
|
||||
kv_cache,
|
||||
block_size,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
q_fp8_out,
|
||||
q_scale,
|
||||
)
|
||||
|
||||
|
||||
def fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
k_norm_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
rotary_dim: int,
|
||||
eps: float,
|
||||
index_q_norm_weight: torch.Tensor | None = None,
|
||||
index_k_norm_weight: torch.Tensor | None = None,
|
||||
num_index_heads: int = 0,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
index_slot_mapping: torch.Tensor | None = None,
|
||||
kv_cache: torch.Tensor | None = None,
|
||||
index_cache: torch.Tensor | None = None,
|
||||
block_size: int = 0,
|
||||
q_out: torch.Tensor | None = None,
|
||||
index_q_out: torch.Tensor | None = None,
|
||||
kv_cache_dtype: str = "auto",
|
||||
skip_index_branch: bool = False,
|
||||
q_fp8_out: torch.Tensor | None = None,
|
||||
q_fp8_scale: float = 1.0,
|
||||
kv_k_scale: torch.Tensor | None = None,
|
||||
kv_v_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Fused MiniMax-M3 attention pre-processing (in-place).
|
||||
|
||||
Applies Gemma RMSNorm + partial NeoX RoPE to ``qkv`` in place. ``qkv`` is a
|
||||
single fused tensor:
|
||||
|
||||
- dense layer (``num_index_heads == 0``): ``[q | k | v]``;
|
||||
- sparse layer (``num_index_heads > 0``): ``[q | k | v | index_q |
|
||||
index_k]`` — the index branch is read straight out of ``qkv``.
|
||||
|
||||
When ``kv_cache`` is given (sparse serving), also scatter-inserts the
|
||||
normed/roped k & v into the paged KV cache by ``slot_mapping`` and the
|
||||
index key into ``index_cache`` by ``index_slot_mapping``. ``kv_cache_dtype``
|
||||
selects the cache storage/conversion path. If
|
||||
``index_slot_mapping`` is omitted, ``slot_mapping`` is used for both caches.
|
||||
|
||||
If ``q_out`` / ``index_q_out`` (contiguous ``[N, nq*128]`` / ``[N,
|
||||
niq*128]``) are given, the normed/roped q / index_q are written there
|
||||
instead of in place — folding the de-interleave into this kernel's store so
|
||||
callers skip a separate ``.contiguous()`` copy before the SM100 sparse
|
||||
attention's flat TMA descriptor.
|
||||
|
||||
If ``q_fp8_out`` is given, the same normalized q is also written in FP8
|
||||
E4M3 using ``q_fp8_scale`` as its dequantization scale. Without ``q_out``,
|
||||
q is then written only in FP8 and the q slice of ``qkv`` is left as is.
|
||||
|
||||
``kv_cache_dtype="nvfp4"`` quantizes k/v into the packed HND
|
||||
``[num_blocks, 2 * num_kv_heads, block_size, 72]`` uint8 cache using the
|
||||
device dequantization scales ``kv_k_scale``/``kv_v_scale``. Slot
|
||||
``2 * head + side`` (K = 0, V = 1) holds that head's E2M1 data followed by
|
||||
its E4M3 block scales, so every head is one contiguous run of the page.
|
||||
|
||||
When ``skip_index_branch`` is true, sparse rows still keep their packed
|
||||
``[index_q | index_k]`` tail, but the kernel only processes the main q/k/v
|
||||
branches and main KV cache. This is used by MiniMax-M3 index-topk reuse
|
||||
layers that consume top-k block ids selected by an earlier sparse layer.
|
||||
"""
|
||||
torch.ops._C.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
q_norm_weight,
|
||||
k_norm_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
index_q_norm_weight,
|
||||
index_k_norm_weight,
|
||||
num_index_heads,
|
||||
slot_mapping,
|
||||
index_slot_mapping,
|
||||
kv_cache,
|
||||
index_cache,
|
||||
block_size,
|
||||
q_out,
|
||||
index_q_out,
|
||||
kv_cache_dtype,
|
||||
skip_index_branch,
|
||||
q_fp8_out,
|
||||
q_fp8_scale,
|
||||
kv_k_scale,
|
||||
kv_v_scale,
|
||||
)
|
||||
|
||||
|
||||
def fused_kda_decode(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -2922,35 +2720,6 @@ def concat_and_cache_mla_grouped(
|
||||
)
|
||||
|
||||
|
||||
def kimi_k3_attn_res(
|
||||
prefix: torch.Tensor,
|
||||
delta: torch.Tensor | None,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
output_norm_weight: torch.Tensor | None,
|
||||
num_blocks: int,
|
||||
block_write_idx: int,
|
||||
eps: float,
|
||||
output_norm_eps: float,
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty_like(prefix)
|
||||
torch.ops._C.kimi_k3_attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
output,
|
||||
num_blocks,
|
||||
block_write_idx,
|
||||
eps,
|
||||
output_norm_eps,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def concat_and_cache_mla_rope_fused(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
@@ -3435,24 +3204,6 @@ def sm100_cutlass_mla_get_workspace_size(
|
||||
)
|
||||
|
||||
|
||||
def dsv3_fused_a_gemm(
|
||||
output: torch.Tensor,
|
||||
mat_a: torch.Tensor,
|
||||
mat_b: torch.Tensor,
|
||||
enable_pdl: bool = False,
|
||||
) -> None:
|
||||
"""Low-latency fused-A-style GEMM (SM 9.0+, BF16, 1-16 tokens).
|
||||
|
||||
Computes ``output = mat_a @ mat_b`` for the compiled Kimi K3 and
|
||||
DeepSeek V3 projection shapes. ``mat_a`` and ``output`` are row-major;
|
||||
``mat_b`` is the column-major transposed weight. ``enable_pdl`` permits
|
||||
programmatic dependent launch for callers that have validated it.
|
||||
|
||||
Requires SM 9.0+.
|
||||
"""
|
||||
torch.ops._C.dsv3_fused_a_gemm(output, mat_a, mat_b, enable_pdl)
|
||||
|
||||
|
||||
def weight_packed_linear_cpu(
|
||||
x: torch.Tensor,
|
||||
packed_weight: torch.Tensor,
|
||||
@@ -4841,24 +4592,3 @@ if hasattr(torch.ops._C, "hadacore_transform"):
|
||||
@register_fake("_C::hadacore_transform")
|
||||
def _hadacore_transform_fake(x: torch.Tensor, inplace: bool) -> torch.Tensor:
|
||||
return torch.empty_like(x) if not inplace else x
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "minimax_allreduce_rms_qk"):
|
||||
|
||||
@register_fake("_C::minimax_allreduce_rms_qk")
|
||||
def _minimax_allreduce_rms_qk_fake(
|
||||
qkv: torch.Tensor,
|
||||
norm_weight_q: torch.Tensor,
|
||||
norm_weight_k: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
rank: int,
|
||||
nranks: int,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
token_num = qkv.shape[0]
|
||||
return (
|
||||
torch.empty([token_num, q_size], dtype=qkv.dtype, device=qkv.device),
|
||||
torch.empty([token_num, kv_size], dtype=qkv.dtype, device=qkv.device),
|
||||
)
|
||||
|
||||
+3
-42
@@ -86,16 +86,7 @@ _REGISTERED_MODEL_CLASS_OVERRIDES: set[tuple[str, str]] = set()
|
||||
RunnerOption = Literal["auto", RunnerType]
|
||||
ConvertType = Literal["none", "embed", "classify"]
|
||||
ConvertOption = Literal["auto", ConvertType]
|
||||
TokenizerMode = Literal[
|
||||
"auto",
|
||||
"hf",
|
||||
"mistral",
|
||||
"deepseek_v32",
|
||||
"deepseek_v4",
|
||||
"inkling",
|
||||
"kimi_k3",
|
||||
"cohere",
|
||||
]
|
||||
TokenizerMode = Literal["auto", "hf"]
|
||||
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
|
||||
LogprobsMode = Literal[
|
||||
"raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
|
||||
@@ -172,24 +163,14 @@ class ModelConfig:
|
||||
tokenizer_mode: TokenizerMode | str = "auto"
|
||||
"""Tokenizer mode:
|
||||
|
||||
- "auto" will use the tokenizer from `mistral_common` for Mistral models
|
||||
if available, otherwise it will use the "hf" tokenizer.
|
||||
- "auto" will use the "hf" tokenizer.
|
||||
- "hf" will use the fast tokenizer if available.
|
||||
- "mistral" will always use the tokenizer from `mistral_common`.
|
||||
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
|
||||
- "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
|
||||
- "deepseek_v41" will use the DeepSeek V4.1 prompt encoder.
|
||||
- "kimi_k3" will always use the "hf" tokenizer but render chat prompts
|
||||
with Kimi K3's Python XTML encoding instead of a Jinja template.
|
||||
- "cohere" uses the standard HF tokenizer but renders the chat template
|
||||
via the `cohere_melody` library (cmd3 / cmd4 templates) instead of
|
||||
Jinja, and surfaces grounded-citation metadata on responses.
|
||||
- Other custom values can be supported via plugins.
|
||||
|
||||
To swap the Rust BPE backend that powers HF fast tokenizers for the
|
||||
[fastokens](https://github.com/crusoecloud/fastokens) implementation, set
|
||||
`VLLM_USE_FASTOKENS=1` instead — that override applies to any mode that
|
||||
loads an HF fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, …)."""
|
||||
loads an HF fast tokenizer."""
|
||||
trust_remote_code: bool = False
|
||||
"""Trust remote code (e.g., from HuggingFace) when downloading the model
|
||||
and tokenizer."""
|
||||
@@ -267,8 +248,6 @@ class ModelConfig:
|
||||
(linear, moe) and ignore patterns; see :class:`QuantizationConfigArgs`.
|
||||
Auto-populated from the matching online shorthand when `quantization` is
|
||||
one of the values in `ONLINE_QUANT_SHORTHAND_NAMES`."""
|
||||
allow_deprecated_quantization: bool = False
|
||||
"""Whether to allow deprecated quantization methods."""
|
||||
enforce_eager: bool = False
|
||||
"""Whether to always use eager-mode PyTorch. If True, we will disable CUDA
|
||||
graph and always execute the model in eager mode. If False, we will use
|
||||
@@ -403,8 +382,6 @@ class ModelConfig:
|
||||
logits_processors: list[str | type[LogitsProcessor]] | None = None
|
||||
"""One or more logits processors' fully-qualified class names or class
|
||||
definitions"""
|
||||
io_processor_plugin: str | None = None
|
||||
"""IOProcessor plugin name to load at model startup"""
|
||||
renderer_num_workers: int = 1
|
||||
"""Number of worker threads in the renderer thread pool. The pool is
|
||||
consumed by the async renderer path (e.g. the OpenAI-compatible API
|
||||
@@ -483,7 +460,6 @@ class ModelConfig:
|
||||
"hf_token",
|
||||
"hf_overrides",
|
||||
"logits_processors",
|
||||
"io_processor_plugin",
|
||||
"pooler_config",
|
||||
"multimodal_config",
|
||||
"limit_mm_per_prompt",
|
||||
@@ -1440,21 +1416,6 @@ class ModelConfig:
|
||||
)
|
||||
current_platform.verify_quantization(self.quantization)
|
||||
|
||||
if self.quantization in me_quant.DEPRECATED_QUANTIZATION_METHODS:
|
||||
if self.allow_deprecated_quantization:
|
||||
logger.warning(
|
||||
"The quantization method %s is deprecated "
|
||||
"and will be removed in future versions of vLLM.",
|
||||
self.quantization,
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"The quantization method %s is deprecated "
|
||||
"and will be removed in future versions of vLLM. To bypass, "
|
||||
"set `--allow-deprecated-quantization`.",
|
||||
self.quantization,
|
||||
)
|
||||
|
||||
def _verify_cuda_graph(self) -> None:
|
||||
# CUDAGraph capture not supported for encoder-decoder models on ROCm
|
||||
unsupported_rocm = self.is_encoder_decoder
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from cutlass import BFloat16, Constexpr, Float32, Uint16, Uint32, cute
|
||||
from cutlass._mlir import ir
|
||||
from cutlass._mlir.dialects import llvm, vector
|
||||
from cutlass.cutlass_dsl import T, dsl_user_op
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp32x2_to_bf16x2(a: Float32, b: Float32, *, loc=None, ip=None) -> Uint32:
|
||||
out = llvm.inline_asm(
|
||||
T.i32(),
|
||||
[a.ir_value(loc=loc, ip=ip), b.ir_value(loc=loc, ip=ip)],
|
||||
"cvt.rn.bf16x2.f32 $0, $2, $1;",
|
||||
"=r,f,f",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
)
|
||||
return Uint32(out)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]:
|
||||
if isinstance(data, Uint32):
|
||||
out = llvm.inline_asm(
|
||||
llvm.StructType.get_literal([T.f32(), T.f32()]),
|
||||
[data.ir_value(loc=loc, ip=ip)],
|
||||
"shl.b32 $0, $2, 16;\n\tand.b32 $1, $2, 0xFFFF0000;",
|
||||
"=f,=f,r",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return (
|
||||
Float32(llvm.extractvalue(T.f32(), out, [0], loc=loc, ip=ip)),
|
||||
Float32(llvm.extractvalue(T.f32(), out, [1], loc=loc, ip=ip)),
|
||||
)
|
||||
|
||||
elif isinstance(data, (cute.Tensor, cute.TensorSSA)):
|
||||
# recast to 32-bit registers if needed
|
||||
if data.element_type == BFloat16:
|
||||
data = cute.recast_tensor(data, Uint32)
|
||||
assert data.element_type == Uint32
|
||||
|
||||
# NOTE: the output is always 1D
|
||||
size = cute.size(data.shape)
|
||||
out = cute.make_rmem_tensor(size * 2, Float32)
|
||||
for i in range(size):
|
||||
out[i * 2], out[i * 2 + 1] = bf16x2_to_fp32x2(data[i])
|
||||
return out
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported type {type(data)}")
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
|
||||
# there is only fp8->fp16 conversion, hence we need to go
|
||||
# round trip through fp16.
|
||||
out = llvm.inline_asm(
|
||||
llvm.StructType.get_literal([T.i32()] * 2),
|
||||
[x.ir_value(loc=loc, ip=ip)],
|
||||
"{\n\t"
|
||||
".reg .b16 x0, x1;\n\t"
|
||||
".reg .b16 t00, t01, t10, t11;\n\t"
|
||||
"mov.b32 {x0, x1}, $2;\n\t"
|
||||
"cvt.rn.f16x2.e4m3x2 $0, x0;\n\t"
|
||||
"cvt.rn.f16x2.e4m3x2 $1, x1;\n\t"
|
||||
"mov.b32 {t00, t01}, $0;\n\t"
|
||||
"mov.b32 {t10, t11}, $1;\n\t"
|
||||
"cvt.rn.bf16.f16 t00, t00;\n\t"
|
||||
"cvt.rn.bf16.f16 t01, t01;\n\t"
|
||||
"cvt.rn.bf16.f16 t10, t10;\n\t"
|
||||
"cvt.rn.bf16.f16 t11, t11;\n\t"
|
||||
"mov.b32 $0, {t00, t01};\n\t"
|
||||
"mov.b32 $1, {t10, t11};\n\t"
|
||||
"}\n",
|
||||
"=r,=r,r",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
)
|
||||
vec = vector.from_elements(
|
||||
ir.VectorType.get([2], T.i32(), loc=loc),
|
||||
[llvm.extractvalue(T.i32(), out, [i], loc=loc, ip=ip) for i in range(2)],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(vec, 2, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp32x2_to_fp8x2(a0: Float32, a1: Float32, *, loc=None, ip=None) -> Uint16:
|
||||
out = llvm.inline_asm(
|
||||
T.i16(),
|
||||
[
|
||||
a0.ir_value(loc=loc, ip=ip),
|
||||
a1.ir_value(loc=loc, ip=ip),
|
||||
],
|
||||
"cvt.rn.satfinite.e4m3x2.f32 $0, $2, $1;",
|
||||
"=h,f,f",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
)
|
||||
return Uint16(out)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp8x4_to_fp16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
|
||||
out = llvm.inline_asm(
|
||||
llvm.StructType.get_literal([T.i32()] * 2),
|
||||
[x.ir_value(loc=loc, ip=ip)],
|
||||
"{\n\t"
|
||||
".reg .b16 lo, hi;\n\t"
|
||||
"mov.b32 {lo, hi}, $2;\n\t"
|
||||
"cvt.rn.f16x2.e4m3x2 $0, lo;\n\t"
|
||||
"cvt.rn.f16x2.e4m3x2 $1, hi;\n\t"
|
||||
"}\n",
|
||||
"=r,=r,r",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
vec = vector.from_elements(
|
||||
ir.VectorType.get([2], T.i32(), loc=loc),
|
||||
[llvm.extractvalue(T.i32(), out, [i], loc=loc, ip=ip) for i in range(2)],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(vec, 2, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp32x4_to_fp8x4(
|
||||
a0: Float32,
|
||||
a1: Float32,
|
||||
a2: Float32,
|
||||
a3: Float32,
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> Uint32:
|
||||
# Pack four FP32 values into one b32 of four e4m3 bytes, byte order
|
||||
# {a0, a1, a2, a3} from low to high address.
|
||||
out = llvm.inline_asm(
|
||||
T.i32(),
|
||||
[
|
||||
a0.ir_value(loc=loc, ip=ip),
|
||||
a1.ir_value(loc=loc, ip=ip),
|
||||
a2.ir_value(loc=loc, ip=ip),
|
||||
a3.ir_value(loc=loc, ip=ip),
|
||||
],
|
||||
"{\n\t"
|
||||
".reg .b16 t0, t1;\n\t"
|
||||
"cvt.rn.satfinite.e4m3x2.f32 t0, $2, $1;\n\t"
|
||||
"cvt.rn.satfinite.e4m3x2.f32 t1, $4, $3;\n\t"
|
||||
"mov.b32 $0, {t0, t1};\n\t"
|
||||
"}\n",
|
||||
"=r,f,f,f,f",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
)
|
||||
return Uint32(out)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp32x8_to_fp4x8(
|
||||
vals: cute.Tensor,
|
||||
offset: Constexpr[int],
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> Uint32:
|
||||
# Pack eight scaled FP32 values into four E2M1x2 bytes, returned as one b32.
|
||||
assert vals.element_type is Float32
|
||||
out = llvm.inline_asm(
|
||||
T.i32(),
|
||||
[vals[offset + i].ir_value(loc=loc, ip=ip) for i in range(8)],
|
||||
"{\n\t"
|
||||
".reg .b8 x0, x1, x2, x3;\n\t"
|
||||
"cvt.rn.satfinite.e2m1x2.f32 x0, $2, $1;\n\t"
|
||||
"cvt.rn.satfinite.e2m1x2.f32 x1, $4, $3;\n\t"
|
||||
"cvt.rn.satfinite.e2m1x2.f32 x2, $6, $5;\n\t"
|
||||
"cvt.rn.satfinite.e2m1x2.f32 x3, $8, $7;\n\t"
|
||||
"mov.b32 $0, {x0, x1, x2, x3};\n\t"
|
||||
"}\n",
|
||||
"=r,f,f,f,f,f,f,f,f",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
)
|
||||
return Uint32(out)
|
||||
@@ -1,47 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from cutlass import Int32
|
||||
from cutlass._mlir.dialects import nvvm
|
||||
from cutlass.cutlass_dsl import dsl_user_op
|
||||
|
||||
_SPACE_MAP = {
|
||||
"cta": nvvm.MBarrierSpaceKind.CTA,
|
||||
"cluster": nvvm.MBarrierSpaceKind.CLUSTER,
|
||||
}
|
||||
|
||||
_MEMORY_ORDER_MAP = {
|
||||
"weak": nvvm.MemOrderKind.WEAK,
|
||||
"relaxed": nvvm.MemOrderKind.RELAXED,
|
||||
"acquire": nvvm.MemOrderKind.ACQUIRE,
|
||||
"release": nvvm.MemOrderKind.RELEASE,
|
||||
"acq_rel": nvvm.MemOrderKind.ACQ_REL,
|
||||
}
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def arrive(mbar, space: str = "cta", order: str = "relaxed", *, loc=None, ip=None):
|
||||
nvvm.mbarrier_txn(
|
||||
mbar.to_llvm_ptr(loc=loc, ip=ip),
|
||||
Int32(1).ir_value(loc=loc, ip=ip),
|
||||
kind=nvvm.MBarrierTxnKind.ARRIVE,
|
||||
space=_SPACE_MAP[space],
|
||||
order=_MEMORY_ORDER_MAP[order],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def arrive_expect_tx(
|
||||
mbar, size, space: str = "cta", order: str = "relaxed", *, loc=None, ip=None
|
||||
):
|
||||
nvvm.mbarrier_txn(
|
||||
mbar.to_llvm_ptr(loc=loc, ip=ip),
|
||||
Int32(size).ir_value(loc=loc, ip=ip),
|
||||
kind=nvvm.MBarrierTxnKind.ARRIVE_EXPECT_TX,
|
||||
space=_SPACE_MAP[space],
|
||||
order=_MEMORY_ORDER_MAP[order],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
@@ -1,635 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import fcntl
|
||||
import os
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from functools import partial
|
||||
from importlib.metadata import version
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from packaging.version import Version
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.parallel_state import get_tensor_model_parallel_rank
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def extract_from_kv_cache(
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
num_tokens: int,
|
||||
) -> torch.Tensor:
|
||||
"""Extract data from KV cache."""
|
||||
block_size = kv_cache.shape[2]
|
||||
return kv_cache[slot_mapping // block_size, :, slot_mapping % block_size][
|
||||
:num_tokens
|
||||
]
|
||||
|
||||
|
||||
def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
|
||||
"""Load hidden states written by ExampleHiddenStatesConnector.
|
||||
|
||||
Blocks (without polling) until the async write is complete by
|
||||
acquiring a shared flock on the companion lock file. The kernel
|
||||
puts the caller to sleep until the writer releases its exclusive lock.
|
||||
|
||||
Args:
|
||||
path: The file path returned in kv_transfer_params["hidden_states_path"].
|
||||
|
||||
Returns:
|
||||
Dict with "hidden_states" and "token_ids" tensors.
|
||||
|
||||
"""
|
||||
lock_path = path + ".lock"
|
||||
with open(lock_path) as lf:
|
||||
fcntl.flock(lf, fcntl.LOCK_SH) # sleeps until writer releases LOCK_EX
|
||||
data = load_file(path, device="cpu")
|
||||
return data
|
||||
|
||||
|
||||
def cleanup_hidden_states(path: str, keep_hidden_states: bool = False) -> None:
|
||||
"""Clean up hidden states file and lock file after loading.
|
||||
|
||||
If keep_hidden_states is True, only removes the lock file
|
||||
and keeps the hidden states file.
|
||||
"""
|
||||
lock_path = path + ".lock"
|
||||
if os.path.exists(lock_path):
|
||||
os.remove(lock_path)
|
||||
if not keep_hidden_states and os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PendingSave:
|
||||
req_id: str
|
||||
filename: str
|
||||
token_ids: torch.Tensor
|
||||
block_ids: list[int]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExampleHiddenStatesConnectorMetadata(KVConnectorMetadata):
|
||||
pending_saves: list[PendingSave] = field(default_factory=list)
|
||||
# req_id → filename for newly scheduled requests — the worker pre-creates
|
||||
# lock files for these so the lock exists before the client receives the
|
||||
# output path.
|
||||
new_req_filenames: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""Simple debug implementation of a HiddenStatesConnector.
|
||||
|
||||
Simply extracts the hidden states from the kv cache and stores them to disk.
|
||||
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
|
||||
"""
|
||||
|
||||
def get_finished_count(self) -> int:
|
||||
"""Number of ``finished_sending`` notifications expected per request.
|
||||
|
||||
Only TP rank 0 writes hidden states to disk, so each request yields a
|
||||
single notification. The default (``None``) makes ``KVOutputAggregator``
|
||||
wait for every worker in the world size; since ranks 1..N-1 never submit
|
||||
a copy, the request would never be reported as finished and its blocks
|
||||
would be leaked forever (the scheduler then spins on
|
||||
``has_finished_requests()``).
|
||||
"""
|
||||
return 1
|
||||
|
||||
@classmethod
|
||||
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
|
||||
"""Index of the KV cache group holding the extracted hidden states.
|
||||
|
||||
Located by spec type so it resolves on both scheduler and worker side.
|
||||
"""
|
||||
if kv_cache_config is None:
|
||||
return 0
|
||||
|
||||
from vllm.v1.kv_cache_interface import HiddenStateCacheSpec
|
||||
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
group_ids = [
|
||||
gid
|
||||
for gid, group in enumerate(groups)
|
||||
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
|
||||
]
|
||||
if len(group_ids) == 1:
|
||||
return group_ids[0]
|
||||
if not group_ids and len(groups) == 1:
|
||||
return 0
|
||||
raise ValueError(
|
||||
"Could not uniquely identify the extract-hidden-states KV cache "
|
||||
f"group among {len(groups)} groups; the hidden-states layer must be "
|
||||
"isolated in its own group (MLA verifiers are unsupported)."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_cache_block_size(
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_config: "KVCacheConfig | None",
|
||||
cache_kv_group_id: int,
|
||||
) -> int:
|
||||
"""Block size of the hidden-states group, read from its own spec.
|
||||
|
||||
cache_config.block_size is bumped to a common multiple for hybrid
|
||||
verifiers; the page-aligned hidden-states group keeps a smaller one.
|
||||
"""
|
||||
if kv_cache_config is None:
|
||||
return vllm_config.cache_config.block_size
|
||||
cache_group = kv_cache_config.kv_cache_groups[cache_kv_group_id]
|
||||
return cache_group.kv_cache_spec.block_size
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
role=role,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
# Read the hidden-states group and its block size from the group spec;
|
||||
# cache_config.block_size is bumped (wrong) for hybrid verifiers.
|
||||
self._cache_kv_group_id = self._find_cache_kv_group_id(kv_cache_config)
|
||||
self._block_size = self._get_cache_block_size(
|
||||
vllm_config, kv_cache_config, self._cache_kv_group_id
|
||||
)
|
||||
self._storage_path = self._kv_transfer_config.get_from_extra_config(
|
||||
"shared_storage_path", "/tmp"
|
||||
)
|
||||
self.cache_layers: list[str] = [] # set by self.register_kv_caches
|
||||
logger.info(self._kv_transfer_config)
|
||||
logger.info("Shared storage path is %s", self._storage_path)
|
||||
|
||||
if Version(version("safetensors")) < Version("0.8.0"):
|
||||
logger.warning(
|
||||
"safetensors < 0.8.0 holds the GIL during save_file, which "
|
||||
"serializes the writer thread pool and hurts throughput. "
|
||||
"Upgrade to safetensors >= 0.8.0 for better performance."
|
||||
)
|
||||
|
||||
assert self._vllm_config.speculative_config is not None, (
|
||||
"ExampleHiddenStatesConnector only works when using "
|
||||
"'extract_hidden_states' speculative method"
|
||||
)
|
||||
spec_config = self._vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.num_hidden_states = len(
|
||||
getattr(spec_config, "eagle_aux_hidden_state_layer_ids", [])
|
||||
)
|
||||
|
||||
# Scheduler-side state
|
||||
self._pending_saves: dict[str, PendingSave] = {}
|
||||
self._request_filenames: dict[str, str] = {}
|
||||
|
||||
# Worker-side state (set by register_kv_caches).
|
||||
self._kv_cache: torch.Tensor | None = None
|
||||
|
||||
# Only TP rank 0 writes hidden states to disk; other TP ranks no-op.
|
||||
# Set in register_kv_caches (after distributed init).
|
||||
self._is_tp_rank_zero: bool = True
|
||||
|
||||
# Async write infrastructure (worker-side).
|
||||
# Dedicated CUDA stream for DtoH copies so they don't block
|
||||
# the default stream (model forward). Thread pool for disk writes.
|
||||
self._copy_stream: torch.cuda.Stream | None = None # lazy init
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=self._kv_transfer_config.get_from_extra_config(
|
||||
"num_writer_threads", 8
|
||||
),
|
||||
thread_name_prefix="vllm-hs-save",
|
||||
)
|
||||
# Whether to use a filesystem lock when writing files to shared storage.
|
||||
# This is necessary for online transfer clients to avoid incomplete reads,
|
||||
# but can be disabled for offline tasks that run tasks in batches to completion
|
||||
self.allow_custom_save_path = self._kv_transfer_config.get_from_extra_config(
|
||||
"allow_custom_save_path", False
|
||||
)
|
||||
if self.allow_custom_save_path:
|
||||
logger.warning(
|
||||
"allow_custom_save_path is enabled. API clients can write "
|
||||
"hidden states to arbitrary paths on the server filesystem. "
|
||||
"Only enable this with trusted clients."
|
||||
)
|
||||
self.use_lock = self._kv_transfer_config.get_from_extra_config(
|
||||
"use_synchronization_lock", True
|
||||
)
|
||||
# req_id → open fd on the .lock file with LOCK_EX held.
|
||||
# Pre-created in wait_for_save when a request first arrives,
|
||||
# consumed by _submit_async_write which passes the fd to the
|
||||
# thread pool worker for release after writing.
|
||||
self._lock_fds: dict[str, int] = {}
|
||||
# req_id → in-flight disk-write Future for that req_id.
|
||||
self._req_futures: dict[str, Future] = {}
|
||||
# req_id → CUDA event marking completion of the DtoH copy. Once
|
||||
# this event is complete the request is considered "done sending"
|
||||
# by get_finished; clients block on the per-file flock to wait for
|
||||
# the disk write itself.
|
||||
self._req_copy_events: dict[str, torch.cuda.Event] = {}
|
||||
# req_ids reported as finished-generating by the scheduler,
|
||||
# accumulated across get_finished calls.
|
||||
self._accumulated_finished_req_ids: set[str] = set()
|
||||
|
||||
def _get_copy_stream(self) -> torch.cuda.Stream:
|
||||
"""Lazily create the copy stream (CUDA must be initialized)."""
|
||||
if self._copy_stream is None:
|
||||
self._copy_stream = torch.cuda.Stream()
|
||||
return self._copy_stream
|
||||
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def start_load_kv(self, *args, **kwargs: Any) -> None:
|
||||
pass # Store-only connector — nothing to load
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
pass # Store-only connector — nothing to load
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
"""Pre-create lock files for newly arrived requests.
|
||||
|
||||
This runs on the worker BEFORE the scheduler returns the output
|
||||
path to the client, guaranteeing that the lock file exists (and
|
||||
LOCK_EX is held) by the time the client tries to open it.
|
||||
"""
|
||||
if not self._is_tp_rank_zero:
|
||||
return
|
||||
if not self.use_lock or not self.has_connector_metadata():
|
||||
return
|
||||
metadata = self._get_connector_metadata()
|
||||
if not isinstance(metadata, ExampleHiddenStatesConnectorMetadata):
|
||||
return
|
||||
for req_id, filename in metadata.new_req_filenames.items():
|
||||
if req_id in self._lock_fds:
|
||||
continue
|
||||
lock_path = filename + ".lock"
|
||||
os.makedirs(os.path.dirname(lock_path), exist_ok=True)
|
||||
lock_fd = os.open(lock_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644)
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX)
|
||||
self._lock_fds[req_id] = lock_fd
|
||||
|
||||
def _on_write_done(self, req_id: str, future: Future) -> None:
|
||||
"""Surface any exception from the disk-write thread and drop the
|
||||
completed future from the in-flight tracking dict."""
|
||||
self._req_futures.pop(req_id, None)
|
||||
exc = future.exception()
|
||||
if exc is not None:
|
||||
logger.error("Hidden-states write failed for req_id=%s: %r", req_id, exc)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
# Delay tp rank0 initialization until after distributed init
|
||||
self._is_tp_rank_zero = get_tensor_model_parallel_rank() == 0
|
||||
|
||||
from vllm.model_executor.models.extract_hidden_states import (
|
||||
CacheOnlyAttentionLayer,
|
||||
)
|
||||
|
||||
# Filter layers to only include CacheOnlyAttentionLayers
|
||||
layers = get_layers_from_vllm_config(
|
||||
self._vllm_config, CacheOnlyAttentionLayer, list(kv_caches.keys())
|
||||
)
|
||||
self.cache_layers = list(layers.keys())
|
||||
assert len(self.cache_layers) == 1, (
|
||||
f"Expected 1 CacheOnlyAttentionLayer, got {len(self.cache_layers)}"
|
||||
)
|
||||
self._kv_cache = kv_caches[self.cache_layers[0]]
|
||||
|
||||
# Block size must match the indexed buffer, else reads hit the wrong
|
||||
# slots. Raise (not assert) so the check survives `python -O`.
|
||||
# Views are [num_blocks, num_heads, block_size, head_size], matching what
|
||||
# extract_from_kv_cache() indexes.
|
||||
if self._block_size != self._kv_cache.shape[2]:
|
||||
raise ValueError(
|
||||
f"Hidden-states block-size mismatch: derived {self._block_size} "
|
||||
f"but buffer block size is {self._kv_cache.shape[2]}; read slots "
|
||||
"would be wrong (likely a hybrid block-size resolution bug)."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _write_tensors(
|
||||
tensors: dict[str, torch.Tensor],
|
||||
event: torch.cuda.Event,
|
||||
filename: str,
|
||||
lock_fd: int | None,
|
||||
) -> None:
|
||||
"""Thread worker: wait for async DtoH copy, write to disk, release lock.
|
||||
|
||||
``lock_fd`` is an open file descriptor on the companion ``.lock``
|
||||
file with ``LOCK_EX`` already held. Closing it releases the lock,
|
||||
which unblocks any client sleeping on ``LOCK_SH``.
|
||||
"""
|
||||
try:
|
||||
event.synchronize()
|
||||
save_file(tensors, filename)
|
||||
finally:
|
||||
if lock_fd is not None:
|
||||
os.close(lock_fd) # releases LOCK_EX
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
# Hidden states are already cached by CacheOnlyAttentionLayer during
|
||||
# forward. Extraction happens in get_finished once all tokens are done.
|
||||
pass
|
||||
|
||||
def _submit_async_write(
|
||||
self,
|
||||
pending: PendingSave,
|
||||
) -> None:
|
||||
"""Extract hidden states from KV cache and submit async DtoH + disk write.
|
||||
|
||||
Called from get_finished for each request that has finished generating.
|
||||
"""
|
||||
if not self._is_tp_rank_zero:
|
||||
return
|
||||
assert self._kv_cache is not None
|
||||
|
||||
# Compute slot mapping from block_ids
|
||||
block_ids_t = torch.tensor(pending.block_ids, dtype=torch.long)
|
||||
num_blocks = block_ids_t.shape[0]
|
||||
block_offsets = torch.arange(0, self._block_size, dtype=torch.long)
|
||||
slot_mapping = (
|
||||
block_offsets.reshape((1, self._block_size))
|
||||
+ block_ids_t.reshape((num_blocks, 1)) * self._block_size
|
||||
)
|
||||
slot_mapping = slot_mapping.flatten()
|
||||
|
||||
num_tokens = pending.token_ids.shape[0]
|
||||
|
||||
copy_stream = self._get_copy_stream()
|
||||
|
||||
# Ensure the copy stream sees all prior writes on the default stream.
|
||||
ready_event = torch.cuda.Event()
|
||||
ready_event.record()
|
||||
copy_stream.wait_event(ready_event)
|
||||
|
||||
with torch.cuda.stream(copy_stream):
|
||||
# Move the CPU slot_mapping to GPU on the copy stream so the
|
||||
# implicit H2D inside fancy indexing doesn't sync the default
|
||||
# stream. Deliberate bulk transfer in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
slot_mapping_gpu = slot_mapping.to(
|
||||
device=self._kv_cache.device, non_blocking=True
|
||||
)
|
||||
hidden_states_gpu = extract_from_kv_cache(
|
||||
self._kv_cache, slot_mapping_gpu, num_tokens
|
||||
)
|
||||
# Async DtoH copy into pinned host memory.
|
||||
pinned_hs = torch.empty_like(
|
||||
hidden_states_gpu, device="cpu", pin_memory=True
|
||||
)
|
||||
pinned_hs.copy_(hidden_states_gpu, non_blocking=True)
|
||||
|
||||
# Record completion of this copy on the copy stream.
|
||||
copy_done = torch.cuda.Event()
|
||||
copy_done.record(copy_stream)
|
||||
|
||||
# token_ids is already on CPU (created in request_finished).
|
||||
assert not pending.token_ids.is_cuda, (
|
||||
"Expected token_ids on CPU, got CUDA tensor"
|
||||
)
|
||||
tensors = {
|
||||
"hidden_states": pinned_hs,
|
||||
"token_ids": pending.token_ids.clone(),
|
||||
}
|
||||
|
||||
# Submit to thread pool for disk write.
|
||||
prior = self._req_futures.get(pending.req_id)
|
||||
assert prior is None, "Found another KV transfer request with same req_id!"
|
||||
|
||||
os.makedirs(os.path.dirname(pending.filename), exist_ok=True)
|
||||
|
||||
# Use the pre-created lock fd from wait_for_save (already holds
|
||||
# LOCK_EX). Falls back to creating one here if use_lock is True
|
||||
# but no pre-created fd exists (shouldn't happen in normal flow).
|
||||
lock_fd = self._lock_fds.pop(pending.req_id, None)
|
||||
if lock_fd is None and self.use_lock:
|
||||
lock_path = pending.filename + ".lock"
|
||||
lock_fd = os.open(lock_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644)
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX)
|
||||
|
||||
future = self._executor.submit(
|
||||
self._write_tensors, tensors, copy_done, pending.filename, lock_fd
|
||||
)
|
||||
self._req_copy_events[pending.req_id] = copy_done
|
||||
self._req_futures[pending.req_id] = future
|
||||
future.add_done_callback(partial(self._on_write_done, pending.req_id))
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally
|
||||
computed tokens for this request
|
||||
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
# This connector is store-only, so we don't need to load any tokens
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
# Usually used to handle allocation of new blocks for requests that are loading
|
||||
# tokens from connector's external kv cache. We never load from external cache
|
||||
# so this is a no-op.
|
||||
assert num_external_tokens == 0, "This connector is store-only"
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify any fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
meta = ExampleHiddenStatesConnectorMetadata()
|
||||
|
||||
# Transfer pending saves into metadata (scheduler → worker bridge)
|
||||
meta.pending_saves = list(self._pending_saves.values())
|
||||
self._pending_saves.clear()
|
||||
|
||||
# Resolve save paths for new requests and tell the worker so it can
|
||||
# pre-create lock files before the client receives the output path.
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
default_path = os.path.join(
|
||||
self._storage_path, f"{new_req.req_id}.safetensors"
|
||||
)
|
||||
kv_params = (
|
||||
new_req.sampling_params.extra_args.get("kv_transfer_params")
|
||||
if new_req.sampling_params and new_req.sampling_params.extra_args
|
||||
else None
|
||||
) or {}
|
||||
custom_path = kv_params.get("hidden_states_path")
|
||||
if custom_path is not None and not self.allow_custom_save_path:
|
||||
logger.warning(
|
||||
"Request %s provided hidden_states_path but "
|
||||
"allow_custom_save_path is disabled. Ignoring "
|
||||
"custom path and using default.",
|
||||
new_req.req_id,
|
||||
)
|
||||
custom_path = None
|
||||
filename = custom_path or default_path
|
||||
self._request_filenames[new_req.req_id] = filename
|
||||
meta.new_req_filenames[new_req.req_id] = filename
|
||||
|
||||
return meta
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Called exactly once when a request has finished, before its blocks are
|
||||
freed.
|
||||
|
||||
Returns True to delay block freeing until get_finished extracts
|
||||
the hidden states from the KV cache.
|
||||
"""
|
||||
req_id = request.request_id
|
||||
filename = self._request_filenames.pop(req_id, None)
|
||||
if filename is None:
|
||||
# Request was aborted before it was ever scheduled, so
|
||||
# build_connector_meta never recorded a filename for it. There are
|
||||
# no hidden states to save and no blocks worth delaying.
|
||||
self._pending_saves.pop(req_id, None)
|
||||
return False, None
|
||||
kv_params = request.kv_transfer_params or {}
|
||||
if kv_params.get("include_output_tokens", False):
|
||||
# Exclude the final token — it was the model's output, never an
|
||||
# input to a forward pass, so its hidden state is not in the cache.
|
||||
token_ids = torch.tensor(list(request.all_token_ids)[:-1])
|
||||
elif request.prompt_token_ids is not None:
|
||||
token_ids = torch.tensor(request.prompt_token_ids)
|
||||
else:
|
||||
logger.warning(
|
||||
"Request %s has no prompt_token_ids (prompt_embeds only). "
|
||||
"Saved token_ids will be empty.",
|
||||
req_id,
|
||||
)
|
||||
token_ids = torch.tensor([], dtype=torch.long)
|
||||
self._pending_saves[req_id] = PendingSave(
|
||||
req_id=req_id,
|
||||
filename=filename,
|
||||
token_ids=token_ids,
|
||||
block_ids=list(block_ids),
|
||||
)
|
||||
return True, {"hidden_states_path": filename}
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""Extract hidden states and poll DtoH-copy completion.
|
||||
|
||||
On the worker side, connector metadata carries pending saves from the
|
||||
scheduler. For each one we extract from the KV cache and launch an
|
||||
async DtoH copy + thread-pool disk write.
|
||||
|
||||
We then poll accumulated finished req_ids: a request is "done sending"
|
||||
once its DtoH copy event is complete. The subsequent disk write may
|
||||
still be in flight; clients block on the per-file flock to wait for it.
|
||||
"""
|
||||
# Extract and submit async writes for newly finished requests
|
||||
if self.has_connector_metadata():
|
||||
connector_metadata = self._get_connector_metadata()
|
||||
if isinstance(connector_metadata, ExampleHiddenStatesConnectorMetadata):
|
||||
for pending in connector_metadata.pending_saves:
|
||||
self._submit_async_write(pending)
|
||||
|
||||
# Poll for completed DtoH copies
|
||||
self._accumulated_finished_req_ids.update(finished_req_ids)
|
||||
|
||||
done_sending: set[str] = set()
|
||||
for req_id in list(self._accumulated_finished_req_ids):
|
||||
event = self._req_copy_events.get(req_id)
|
||||
if event is not None and not event.query():
|
||||
continue # DtoH copy still in flight
|
||||
self._req_copy_events.pop(req_id, None)
|
||||
self._accumulated_finished_req_ids.discard(req_id)
|
||||
# Only report requests we actually tracked for sending.
|
||||
if event is not None:
|
||||
done_sending.add(req_id)
|
||||
# Clean up any leftover lock fds (e.g. aborted requests that never
|
||||
# went through _submit_async_write).
|
||||
lock_fd = self._lock_fds.pop(req_id, None)
|
||||
if lock_fd is not None:
|
||||
os.close(lock_fd)
|
||||
|
||||
return done_sending or None, None
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
if not 0 <= self._cache_kv_group_id < len(block_ids):
|
||||
# No per-group block table (e.g. a request that was never
|
||||
# scheduled); treat as a no-op instead of indexing out of bounds.
|
||||
return False, None
|
||||
return self.request_finished(request, block_ids[self._cache_kv_group_id])
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
"""Get the required KV cache layout for this connector.
|
||||
|
||||
Args:
|
||||
vllm_config (VllmConfig): the vllm config.
|
||||
|
||||
Returns:
|
||||
str: the required KV cache layout. e.g. HND, or NHD.
|
||||
None if the connector does not require a specific layout.
|
||||
|
||||
"""
|
||||
if cls is KVConnectorBase_V1:
|
||||
raise TypeError(
|
||||
"get_required_kvcache_layout should not be called "
|
||||
"on the abstract base class"
|
||||
)
|
||||
# LBNHC means we have (num_tokens, num_heads)
|
||||
# LBHNC means we have (num_heads, num_tokens)
|
||||
# For now, we only support LBNHC layout since this keeps the
|
||||
# hidden states for each token together in memory.
|
||||
# LBHNC is primarily used when sharding heads across devices.
|
||||
return "LBNHC"
|
||||
@@ -638,7 +638,6 @@ class EngineArgs:
|
||||
:class:`QuantizationConfigArgs`. Auto-populated from the matching online
|
||||
shorthand when `quantization` is one of the values in
|
||||
`ONLINE_QUANT_SHORTHAND_NAMES`."""
|
||||
allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
|
||||
enforce_eager: bool = ModelConfig.enforce_eager
|
||||
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
|
||||
enable_shm_tensor_arena: bool = ParallelConfig.enable_shm_tensor_arena
|
||||
@@ -675,7 +674,6 @@ class EngineArgs:
|
||||
mm_encoder_fp8_scale_save_margin: float = (
|
||||
MultiModalConfig.mm_encoder_fp8_scale_save_margin
|
||||
)
|
||||
io_processor_plugin: str | None = None
|
||||
renderer_num_workers: int = 1
|
||||
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
|
||||
video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
|
||||
@@ -981,10 +979,6 @@ class EngineArgs:
|
||||
model_group.add_argument(
|
||||
"--quantization-config", **model_kwargs["quantization_config"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--allow-deprecated-quantization",
|
||||
**model_kwargs["allow_deprecated_quantization"],
|
||||
)
|
||||
model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
|
||||
model_group.add_argument(
|
||||
"--enable-return-routed-experts",
|
||||
@@ -1053,9 +1047,6 @@ class EngineArgs:
|
||||
model_group.add_argument(
|
||||
"--logits-processors", **model_kwargs["logits_processors"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--io-processor-plugin", **model_kwargs["io_processor_plugin"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--renderer-num-workers",
|
||||
**model_kwargs["renderer_num_workers"],
|
||||
@@ -2000,7 +1991,6 @@ class EngineArgs:
|
||||
max_model_len=self.max_model_len,
|
||||
quantization=self.quantization,
|
||||
quantization_config=self.quantization_config,
|
||||
allow_deprecated_quantization=self.allow_deprecated_quantization,
|
||||
enforce_eager=self.enforce_eager,
|
||||
return_sampling_mask=self.return_sampling_mask,
|
||||
max_logprobs=self.max_logprobs,
|
||||
@@ -2046,7 +2036,6 @@ class EngineArgs:
|
||||
mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
|
||||
mm_device_do_normalize=self.mm_device_do_normalize,
|
||||
mm_processor_device=self.mm_processor_device,
|
||||
io_processor_plugin=self.io_processor_plugin,
|
||||
renderer_num_workers=self.renderer_num_workers,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicCountTokensRequest,
|
||||
AnthropicCountTokensResponse,
|
||||
AnthropicError,
|
||||
AnthropicErrorResponse,
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def messages(request: Request) -> AnthropicServingMessages:
|
||||
return request.app.state.anthropic_serving_messages
|
||||
|
||||
|
||||
def translate_error_response(response: ErrorResponse) -> JSONResponse:
|
||||
anthropic_error = AnthropicErrorResponse(
|
||||
error=AnthropicError(
|
||||
type=response.error.type,
|
||||
message=response.error.message,
|
||||
)
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=response.error.code, content=anthropic_error.model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/messages",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
|
||||
handler = messages(raw_request)
|
||||
if handler is None:
|
||||
base_server = raw_request.app.state.serving_tokenization
|
||||
error = base_server.create_error_response(
|
||||
NotImplementedError("The model does not support Messages API")
|
||||
)
|
||||
return translate_error_response(error)
|
||||
|
||||
try:
|
||||
generator = await handler.create_messages(request, raw_request)
|
||||
except Exception as e:
|
||||
logger.exception("Error in create_messages: %s", e)
|
||||
return translate_error_response(create_error_response(e))
|
||||
|
||||
if isinstance(generator, ErrorResponse):
|
||||
return translate_error_response(generator)
|
||||
|
||||
elif isinstance(generator, AnthropicMessagesResponse):
|
||||
resp = generator.model_dump(exclude_none=True)
|
||||
logger.debug("Anthropic Messages Response: %s", resp)
|
||||
return JSONResponse(content=resp)
|
||||
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/messages/count_tokens",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"model": AnthropicCountTokensResponse},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def count_tokens(request: AnthropicCountTokensRequest, raw_request: Request):
|
||||
handler = messages(raw_request)
|
||||
if handler is None:
|
||||
base_server = raw_request.app.state.serving_tokenization
|
||||
error = base_server.create_error_response(
|
||||
NotImplementedError("The model does not support Messages API")
|
||||
)
|
||||
return translate_error_response(error)
|
||||
|
||||
try:
|
||||
response = await handler.count_tokens(request, raw_request)
|
||||
except Exception as e:
|
||||
logger.exception("Error in count_tokens: %s", e)
|
||||
return translate_error_response(create_error_response(e))
|
||||
|
||||
if isinstance(response, ErrorResponse):
|
||||
return translate_error_response(response)
|
||||
|
||||
return JSONResponse(content=response.model_dump(exclude_none=True))
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -1,451 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pydantic models for Anthropic API protocol."""
|
||||
|
||||
import time
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
MMPayload,
|
||||
resolve_return_mm_kwargs_alias,
|
||||
)
|
||||
|
||||
|
||||
class AnthropicError(BaseModel):
|
||||
"""Error structure for Anthropic API."""
|
||||
|
||||
type: str
|
||||
message: str
|
||||
|
||||
|
||||
class AnthropicErrorResponse(BaseModel):
|
||||
"""Error response structure for Anthropic API."""
|
||||
|
||||
type: Literal["error"] = "error"
|
||||
error: AnthropicError
|
||||
|
||||
|
||||
class AnthropicUsage(BaseModel):
|
||||
"""Token usage information."""
|
||||
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_creation_input_tokens: int | None = None
|
||||
cache_read_input_tokens: int | None = None
|
||||
|
||||
|
||||
class AnthropicContentBlock(BaseModel):
|
||||
"""Content block in message."""
|
||||
|
||||
type: Literal[
|
||||
"text",
|
||||
"image",
|
||||
"tool_use",
|
||||
"tool_result",
|
||||
"tool_reference",
|
||||
"thinking",
|
||||
"redacted_thinking",
|
||||
"tool_addition",
|
||||
"tool_removal",
|
||||
]
|
||||
text: str | None = None
|
||||
# For image content
|
||||
source: dict[str, Any] | None = None
|
||||
# For tool use/result
|
||||
id: str | None = None
|
||||
tool_use_id: str | None = None
|
||||
name: str | None = None
|
||||
input: dict[str, Any] | None = None
|
||||
content: str | list[dict[str, Any]] | None = None
|
||||
is_error: bool | None = None
|
||||
# For tool_reference content
|
||||
tool_name: str | None = None
|
||||
# For thinking content
|
||||
thinking: str | None = None
|
||||
signature: str | None = None
|
||||
# For redacted thinking content (safety-filtered by the API)
|
||||
data: str | None = None
|
||||
# For tool_addition/tool_removal content
|
||||
tool: "AnthropicToolChange | None" = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_tool_change(self) -> "AnthropicContentBlock":
|
||||
if self.type not in ("tool_addition", "tool_removal"):
|
||||
return self
|
||||
if self.tool is None:
|
||||
raise ValueError(f"tool is required for {self.type} blocks")
|
||||
if self.type == "tool_removal" and self.tool.type != "tool_reference":
|
||||
raise ValueError("tool_removal only accepts a tool_reference")
|
||||
return self
|
||||
|
||||
|
||||
def _reject_tool_changes(content: str | list[AnthropicContentBlock] | None) -> None:
|
||||
if content is None or isinstance(content, str):
|
||||
return
|
||||
for block in content:
|
||||
if block.type in ("tool_addition", "tool_removal"):
|
||||
raise ValueError(
|
||||
f'{block.type} blocks are only allowed in messages with role "system"'
|
||||
)
|
||||
|
||||
|
||||
class AnthropicMessage(BaseModel):
|
||||
"""Message structure."""
|
||||
|
||||
role: Literal["user", "assistant", "system"]
|
||||
content: str | list[AnthropicContentBlock]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_tool_change_role(self) -> "AnthropicMessage":
|
||||
if self.role != "system":
|
||||
_reject_tool_changes(self.content)
|
||||
return self
|
||||
|
||||
|
||||
class AnthropicTool(BaseModel):
|
||||
"""Tool definition."""
|
||||
|
||||
name: str
|
||||
description: str | None = None
|
||||
input_schema: dict[str, Any]
|
||||
strict: bool | None = None
|
||||
defer_loading: bool | None = None
|
||||
|
||||
@field_validator("input_schema")
|
||||
@classmethod
|
||||
def validate_input_schema(cls, v):
|
||||
if not isinstance(v, dict):
|
||||
raise ValueError("input_schema must be a dictionary")
|
||||
if "type" not in v:
|
||||
v["type"] = "object" # Default to object type
|
||||
return v
|
||||
|
||||
|
||||
class AnthropicToolChangeReference(BaseModel):
|
||||
"""A tool named by a tool_addition or tool_removal block."""
|
||||
|
||||
type: Literal["tool_reference"]
|
||||
name: str
|
||||
|
||||
|
||||
class AnthropicToolChangeDefinition(BaseModel):
|
||||
"""A tool defined by value in a tool_addition block."""
|
||||
|
||||
type: Literal["tool_definition"]
|
||||
definition: AnthropicTool
|
||||
|
||||
|
||||
AnthropicToolChange = Annotated[
|
||||
AnthropicToolChangeReference | AnthropicToolChangeDefinition,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class AnthropicToolChoice(BaseModel):
|
||||
"""Tool Choice definition."""
|
||||
|
||||
type: Literal["auto", "any", "tool", "none"]
|
||||
name: str | None = None
|
||||
disable_parallel_tool_use: bool | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_name_required_for_tool(self) -> "AnthropicToolChoice":
|
||||
if self.type == "tool" and not self.name:
|
||||
raise ValueError("tool_choice.name is required when type is 'tool'")
|
||||
return self
|
||||
|
||||
|
||||
class AnthropicJsonOutputFormat(BaseModel):
|
||||
"""JSON output format configuration."""
|
||||
|
||||
json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
|
||||
type: Literal["json_schema"] = "json_schema"
|
||||
|
||||
|
||||
AnthropicEffort = Literal["low", "medium", "high", "xhigh", "max"]
|
||||
# reasoning_effort sent to the chat template for thinking.type=disabled.
|
||||
AnthropicDisabledThinkingEffort = Literal["none", AnthropicEffort]
|
||||
AnthropicDisabledThinkingEffortOption = Literal["auto", AnthropicDisabledThinkingEffort]
|
||||
AnthropicThinkingDisplay = Literal["summarized", "omitted", "updates"]
|
||||
|
||||
|
||||
class AnthropicOutputConfig(BaseModel):
|
||||
"""Configuration options for the model's output, such as the output format."""
|
||||
|
||||
effort: AnthropicEffort | None = None
|
||||
format: AnthropicJsonOutputFormat | None = None
|
||||
|
||||
|
||||
class AnthropicThinkingConfigEnabled(BaseModel):
|
||||
"""Extended thinking with a fixed token budget.
|
||||
|
||||
``display`` is accepted but ignored: reasoning is always returned.
|
||||
"""
|
||||
|
||||
type: Literal["enabled"]
|
||||
budget_tokens: int = Field(ge=1024)
|
||||
display: AnthropicThinkingDisplay | None = None
|
||||
|
||||
|
||||
class AnthropicThinkingConfigDisabled(BaseModel):
|
||||
type: Literal["disabled"]
|
||||
|
||||
|
||||
class AnthropicThinkingConfigAdaptive(BaseModel):
|
||||
"""Extended thinking whose depth the model chooses.
|
||||
|
||||
``display`` is accepted but ignored: reasoning is always returned.
|
||||
"""
|
||||
|
||||
type: Literal["adaptive"]
|
||||
display: AnthropicThinkingDisplay | None = None
|
||||
|
||||
|
||||
AnthropicThinkingConfig = Annotated[
|
||||
AnthropicThinkingConfigEnabled
|
||||
| AnthropicThinkingConfigDisabled
|
||||
| AnthropicThinkingConfigAdaptive,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class AnthropicMessagesRequest(BaseModel):
|
||||
"""Anthropic Messages API request."""
|
||||
|
||||
model: str
|
||||
messages: list[AnthropicMessage]
|
||||
max_tokens: int
|
||||
metadata: dict[str, Any] | None = None
|
||||
output_config: AnthropicOutputConfig | None = None
|
||||
thinking: AnthropicThinkingConfig | None = None
|
||||
stop_sequences: (
|
||||
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||
) = None
|
||||
stream: bool | None = False
|
||||
system: str | list[AnthropicContentBlock] | None = None
|
||||
temperature: float | None = None
|
||||
tool_choice: AnthropicToolChoice | None = None
|
||||
tools: list[AnthropicTool] | None = None
|
||||
top_k: int | None = None
|
||||
top_p: float | None = None
|
||||
|
||||
# vLLM-specific fields that are not in Anthropic spec
|
||||
mm_payload: MMPayload = Field(
|
||||
default="tensors",
|
||||
description=(
|
||||
"What the render response's `features` carry for multimodal "
|
||||
"items. 'tensors' serializes the processed tensors into "
|
||||
"`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
|
||||
"callers that need only the token layout and item hashes, such as "
|
||||
"cache-aware routers. Do not send a 'none' response to "
|
||||
"`/inference/v1/generate` which reads a null `kwargs_data` as "
|
||||
"every item being cached. Only supported on the render endpoints. "
|
||||
"Ignored on regular generation endpoints."
|
||||
),
|
||||
)
|
||||
return_mm_kwargs: bool | None = Field(
|
||||
default=None,
|
||||
deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
|
||||
description=(
|
||||
"Deprecated alias: false is the same as `mm_payload='none'`. "
|
||||
"Ignored when `mm_payload` is set."
|
||||
),
|
||||
)
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=1024,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
watermarking: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Whether to apply the engine's configured watermark to this request."
|
||||
),
|
||||
)
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional request parameters with (list of) string or "
|
||||
"numeric values, used by custom extensions."
|
||||
),
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the template."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def validate_model(cls, v):
|
||||
if not v:
|
||||
raise ValueError("Model is required")
|
||||
return v
|
||||
|
||||
@field_validator("max_tokens")
|
||||
@classmethod
|
||||
def validate_max_tokens(cls, v):
|
||||
if v <= 0:
|
||||
raise ValueError("max_tokens must be positive")
|
||||
return v
|
||||
|
||||
@field_validator("system")
|
||||
@classmethod
|
||||
def validate_system(cls, v):
|
||||
_reject_tool_changes(v)
|
||||
return v
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
|
||||
return resolve_return_mm_kwargs_alias(data)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_thinking_budget(self) -> "AnthropicMessagesRequest":
|
||||
# P/D prefill legs are sent with max_tokens=1 and never decode; the
|
||||
# decode leg carries the client's max_tokens and is still checked.
|
||||
if self.kv_transfer_params and self.kv_transfer_params.get("do_remote_decode"):
|
||||
return self
|
||||
if (
|
||||
isinstance(self.thinking, AnthropicThinkingConfigEnabled)
|
||||
and self.thinking.budget_tokens >= self.max_tokens
|
||||
):
|
||||
raise ValueError("thinking.budget_tokens must be less than max_tokens")
|
||||
return self
|
||||
|
||||
|
||||
class AnthropicDelta(BaseModel):
|
||||
"""Delta for streaming responses."""
|
||||
|
||||
type: (
|
||||
Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"]
|
||||
| None
|
||||
) = None
|
||||
text: str | None = None
|
||||
thinking: str | None = None
|
||||
partial_json: str | None = None
|
||||
signature: str | None = None
|
||||
|
||||
# Message delta
|
||||
stop_reason: (
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
|
||||
) = None
|
||||
stop_sequence: str | None = None
|
||||
|
||||
|
||||
class AnthropicStreamEvent(BaseModel):
|
||||
"""Streaming event."""
|
||||
|
||||
type: Literal[
|
||||
"message_start",
|
||||
"message_delta",
|
||||
"message_stop",
|
||||
"content_block_start",
|
||||
"content_block_delta",
|
||||
"content_block_stop",
|
||||
"ping",
|
||||
"error",
|
||||
]
|
||||
message: "AnthropicMessagesResponse | None" = None
|
||||
delta: AnthropicDelta | None = None
|
||||
content_block: AnthropicContentBlock | None = None
|
||||
index: int | None = None
|
||||
error: AnthropicError | None = None
|
||||
usage: AnthropicUsage | None = None
|
||||
|
||||
|
||||
class AnthropicMessagesResponse(BaseModel):
|
||||
"""Anthropic Messages API response."""
|
||||
|
||||
id: str
|
||||
type: Literal["message"] = "message"
|
||||
role: Literal["assistant"] = "assistant"
|
||||
content: list[AnthropicContentBlock]
|
||||
model: str
|
||||
stop_reason: (
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
|
||||
) = None
|
||||
stop_sequence: str | None = None
|
||||
usage: AnthropicUsage | None = None
|
||||
|
||||
# vLLM-specific fields that are not in Anthropic spec
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
|
||||
def model_post_init(self, __context):
|
||||
if not self.id:
|
||||
self.id = f"msg_{int(time.time() * 1000)}"
|
||||
|
||||
|
||||
class AnthropicContextManagement(BaseModel):
|
||||
"""Context management information for token counting."""
|
||||
|
||||
original_input_tokens: int
|
||||
|
||||
|
||||
class AnthropicCountTokensRequest(BaseModel):
|
||||
"""Anthropic messages.count_tokens request."""
|
||||
|
||||
model: str
|
||||
messages: list[AnthropicMessage]
|
||||
output_config: AnthropicOutputConfig | None = None
|
||||
system: str | list[AnthropicContentBlock] | None = None
|
||||
thinking: AnthropicThinkingConfig | None = None
|
||||
tool_choice: AnthropicToolChoice | None = None
|
||||
tools: list[AnthropicTool] | None = None
|
||||
|
||||
# vLLM-specific fields that are not in Anthropic spec
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the template."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def validate_model(cls, v):
|
||||
if not v:
|
||||
raise ValueError("Model is required")
|
||||
return v
|
||||
|
||||
@field_validator("system")
|
||||
@classmethod
|
||||
def validate_system(cls, v):
|
||||
_reject_tool_changes(v)
|
||||
return v
|
||||
|
||||
|
||||
class AnthropicCountTokensResponse(BaseModel):
|
||||
"""Anthropic messages.count_tokens response."""
|
||||
|
||||
input_tokens: int
|
||||
context_management: AnthropicContextManagement | None = None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import inspect
|
||||
|
||||
import uvloop
|
||||
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.render.entry import run_launch_fastapi
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DESCRIPTION = "Launch individual vLLM components."
|
||||
|
||||
|
||||
class LaunchSubcommandBase(CLISubcommand):
|
||||
"""The base class of subcommands for `vllm launch`."""
|
||||
|
||||
help: str
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
"""Add the CLI arguments to the parser.
|
||||
|
||||
By default, uses the subcommand's docstring as the description and adds
|
||||
the standard vLLM serving arguments.
|
||||
Subclasses can override to add component-specific arguments.
|
||||
"""
|
||||
if cls.__doc__:
|
||||
parser.description = inspect.cleandoc(cls.__doc__)
|
||||
make_arg_parser(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RenderSubcommand(LaunchSubcommandBase):
|
||||
"""`vllm launch render` starts a GPU-less rendering server for preprocessing
|
||||
and postprocessing only.
|
||||
|
||||
```bash
|
||||
vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100
|
||||
```
|
||||
|
||||
This command reuses the standard serving parser, so model, frontend,
|
||||
networking, and related CLI options follow the same conventions as
|
||||
[`vllm serve`](https://docs.vllm.ai/en/latest/cli/serve/).
|
||||
"""
|
||||
|
||||
name = "render"
|
||||
help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
uvloop.run(run_launch_fastapi(args))
|
||||
|
||||
|
||||
class LaunchSubcommand(CLISubcommand):
|
||||
"""The `launch` subcommand for the vLLM CLI.
|
||||
|
||||
Uses nested sub-subcommands so each component can define its own
|
||||
arguments independently (e.g. ``vllm launch render``).
|
||||
"""
|
||||
|
||||
name = "launch"
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
if hasattr(args, "model_tag") and args.model_tag is not None:
|
||||
args.model = args.model_tag
|
||||
|
||||
args.launch_command(args)
|
||||
|
||||
def validate(self, args: argparse.Namespace) -> None:
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
def subparser_init(
|
||||
self, subparsers: argparse._SubParsersAction
|
||||
) -> FlexibleArgumentParser:
|
||||
launch_parser = subparsers.add_parser(
|
||||
self.name,
|
||||
help=DESCRIPTION,
|
||||
description=DESCRIPTION,
|
||||
usage=f"vllm {self.name} <component> [options]",
|
||||
)
|
||||
launch_subparsers = launch_parser.add_subparsers(
|
||||
required=True, dest="launch_component"
|
||||
)
|
||||
|
||||
for cmd_cls in LaunchSubcommandBase.__subclasses__():
|
||||
cmd_subparser = launch_subparsers.add_parser(
|
||||
cmd_cls.name,
|
||||
help=cmd_cls.help,
|
||||
usage=f"vllm {self.name} {cmd_cls.name} [options]",
|
||||
)
|
||||
cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)
|
||||
cmd_cls.add_cli_args(cmd_subparser)
|
||||
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
|
||||
subcmd=f"{self.name} {cmd_cls.name}"
|
||||
)
|
||||
|
||||
return launch_parser
|
||||
|
||||
|
||||
def cmd_init() -> list[CLISubcommand]:
|
||||
return [LaunchSubcommand()]
|
||||
@@ -42,10 +42,8 @@ def main():
|
||||
|
||||
import vllm.entrypoints.cli.collect_env
|
||||
import vllm.entrypoints.cli.download_kernels
|
||||
import vllm.entrypoints.cli.launch
|
||||
import vllm.entrypoints.cli.openai
|
||||
import vllm.entrypoints.cli.preload
|
||||
import vllm.entrypoints.cli.run_batch
|
||||
import vllm.entrypoints.cli.serve
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
VLLM_SUBCMD_PARSER_EPILOG,
|
||||
@@ -56,11 +54,9 @@ def main():
|
||||
CMD_MODULES = [
|
||||
vllm.entrypoints.cli.openai,
|
||||
vllm.entrypoints.cli.serve,
|
||||
vllm.entrypoints.cli.launch,
|
||||
vllm.entrypoints.cli.collect_env,
|
||||
vllm.entrypoints.cli.download_kernels,
|
||||
vllm.entrypoints.cli.preload,
|
||||
vllm.entrypoints.cli.run_batch,
|
||||
]
|
||||
|
||||
cli_env_setup()
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import importlib.metadata
|
||||
import typing
|
||||
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RunBatchSubcommand(CLISubcommand):
|
||||
"""The `run-batch` subcommand for vLLM CLI."""
|
||||
|
||||
name = "run-batch"
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
from vllm.entrypoints.launchers.run_batch import main as run_batch_main
|
||||
|
||||
logger.info(
|
||||
"vLLM batch processing API version %s", importlib.metadata.version("vllm")
|
||||
)
|
||||
logger.info("args: %s", args)
|
||||
|
||||
# Start the Prometheus metrics server.
|
||||
# LLMEngine uses the Prometheus client
|
||||
# to publish metrics at the /metrics endpoint.
|
||||
if args.enable_metrics:
|
||||
from prometheus_client import start_http_server
|
||||
|
||||
logger.info("Prometheus metrics enabled")
|
||||
start_http_server(port=args.port, addr=args.url)
|
||||
else:
|
||||
logger.info("Prometheus metrics disabled")
|
||||
|
||||
asyncio.run(run_batch_main(args))
|
||||
|
||||
def subparser_init(
|
||||
self, subparsers: argparse._SubParsersAction
|
||||
) -> FlexibleArgumentParser:
|
||||
from vllm.entrypoints.launchers.run_batch import make_arg_parser
|
||||
|
||||
run_batch_parser = subparsers.add_parser(
|
||||
self.name,
|
||||
help="Run batch prompts and write results to file.",
|
||||
description=(
|
||||
"Run batch prompts using vLLM's OpenAI-compatible API.\n"
|
||||
"Supports local or HTTP input/output files."
|
||||
),
|
||||
usage="vllm run-batch -i INPUT.jsonl -o OUTPUT.jsonl --model <model>",
|
||||
)
|
||||
run_batch_parser = make_arg_parser(run_batch_parser)
|
||||
run_batch_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
|
||||
return run_batch_parser
|
||||
|
||||
|
||||
def cmd_init() -> list[CLISubcommand]:
|
||||
return [RunBatchSubcommand()]
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,309 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FastAPI router for the Cohere Chat v2 API.
|
||||
|
||||
Exposes two routes:
|
||||
|
||||
* ``POST /cohere/v2/chat`` - the chat endpoint itself.
|
||||
* ``POST /cohere/v2/chat/render`` - tokenize a request without running
|
||||
generation, mirroring ``POST /v1/chat/completions/render``.
|
||||
|
||||
The Cohere v2 protocol models are sourced from the official ``cohere``
|
||||
Python SDK (``pip install cohere``). To keep that an *optional*
|
||||
dependency for vLLM, the SDK-dependent imports - and the route handlers
|
||||
themselves - are gated on a one-shot probe at module load. If the SDK
|
||||
isn't installed, :func:`attach_router` becomes a no-op (with an info
|
||||
log) and vLLM continues to boot normally.
|
||||
|
||||
Even when the SDK is installed, :func:`attach_router` also requires
|
||||
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
|
||||
the chat route. The render route additionally requires scale-out
|
||||
endpoints to be enabled, i.e. ``--enable-scale-out`` or
|
||||
``--tokens-only``. These gates keep unrelated deployments from
|
||||
accidentally exposing the APIs.
|
||||
|
||||
Note: the handlers must live at module scope (not inside
|
||||
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
|
||||
the ``CohereChatV2Request`` body annotation against the module's
|
||||
globals. Defining them locally inside ``attach_router`` would hide the
|
||||
type from ``get_type_hints``, causing FastAPI to silently degrade the
|
||||
body parameter into a query parameter and reject every request with
|
||||
422.
|
||||
"""
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
_COHERE_PATH_PREFIX = "/cohere/"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
import cohere # noqa: F401 -- dependency probe
|
||||
except ImportError:
|
||||
_SDK_AVAILABLE = False
|
||||
else:
|
||||
_SDK_AVAILABLE = True
|
||||
|
||||
|
||||
if _SDK_AVAILABLE:
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
CohereChatV2Request,
|
||||
CohereChatV2Response,
|
||||
CohereError,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
|
||||
router = APIRouter()
|
||||
render_router = APIRouter()
|
||||
|
||||
def _serving(request: Request) -> CohereServingChatV2 | None:
|
||||
return getattr(request.app.state, "cohere_serving_chat_v2", None)
|
||||
|
||||
def _serving_render(request: Request) -> ServingRender | None:
|
||||
return getattr(request.app.state, "serving_render", None)
|
||||
|
||||
def _request_id(raw_request: Request | None) -> str | None:
|
||||
"""Best-effort lookup of the active request id.
|
||||
|
||||
Prefers the id the underlying chat handler stamped onto
|
||||
``raw_request.state.request_metadata`` (if it got that far before
|
||||
failing), falling back to the ``X-Request-Id`` HTTP header. May
|
||||
return ``None`` if neither is available, in which case the field
|
||||
is omitted from the response.
|
||||
"""
|
||||
if raw_request is None:
|
||||
return None
|
||||
meta = getattr(raw_request.state, "request_metadata", None)
|
||||
if meta is not None and getattr(meta, "request_id", None):
|
||||
return meta.request_id
|
||||
return raw_request.headers.get("X-Request-Id")
|
||||
|
||||
def _error_response(
|
||||
error: ErrorResponse,
|
||||
raw_request: Request | None,
|
||||
*,
|
||||
fallback_status: int = HTTPStatus.BAD_REQUEST,
|
||||
) -> JSONResponse:
|
||||
"""Translate vLLM's internal error envelope into Cohere's shape."""
|
||||
info = error.error
|
||||
status = info.code or fallback_status
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content=CohereError(
|
||||
message=sanitize_message(info.message),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
@router.post(
|
||||
"/cohere/v2/chat",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def chat_v2(request: CohereChatV2Request, raw_request: Request):
|
||||
handler = _serving(raw_request)
|
||||
if handler is None:
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
|
||||
content=CohereError(
|
||||
message="The model does not support the Cohere v2 chat API.",
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
try:
|
||||
result = await handler.create_chat_v2(request, raw_request)
|
||||
except Exception as e:
|
||||
logger.exception("Error in /cohere/v2/chat: %s", e)
|
||||
return _error_response(create_error_response(e), raw_request)
|
||||
|
||||
match result:
|
||||
case ErrorResponse():
|
||||
return _error_response(result, raw_request)
|
||||
case CohereChatV2Response():
|
||||
return JSONResponse(content=result.model_dump(exclude_none=True))
|
||||
case _:
|
||||
return StreamingResponse(content=result, media_type="text/event-stream")
|
||||
|
||||
@render_router.post(
|
||||
"/cohere/v2/chat/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
async def chat_v2_render(request: CohereChatV2Request, raw_request: Request):
|
||||
"""Tokenize a Cohere v2 chat request without running generation.
|
||||
|
||||
The Cohere counterpart to ``POST /v1/chat/completions/render``.
|
||||
The v2 body goes through the same conversion
|
||||
``/cohere/v2/chat`` uses and is then handed to the shared
|
||||
:class:`ServingRender`, so the returned ``GenerateRequest``
|
||||
carries the prompt tokens and sampling params the chat endpoint
|
||||
would have sent to the engine.
|
||||
"""
|
||||
handler = _serving(raw_request)
|
||||
render_handler = _serving_render(raw_request)
|
||||
if handler is None or render_handler is None:
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
|
||||
content=CohereError(
|
||||
message=(
|
||||
"The model does not support the Cohere v2 chat render API."
|
||||
),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
try:
|
||||
chat_request = handler.to_chat_completion_request(request)
|
||||
result = await render_handler.render_chat_request(chat_request)
|
||||
except Exception as e:
|
||||
logger.exception("Error in /cohere/v2/chat/render: %s", e)
|
||||
return _error_response(create_error_response(e), raw_request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return _error_response(result, raw_request)
|
||||
|
||||
if (kwargs := result.reasoning_parser_kwargs) is not None:
|
||||
kwargs.chat_template_kwargs = handler._engine_chat_template_kwargs(
|
||||
kwargs.chat_template_kwargs
|
||||
)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
|
||||
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
|
||||
|
||||
The endpoint handler above already returns :class:`CohereError` for
|
||||
errors it owns, but globally-registered exception handlers (e.g.
|
||||
:func:`validation_exception_handler` for pydantic body errors,
|
||||
:func:`http_exception_handler`, engine error handlers) fire *before*
|
||||
the handler runs and produce vLLM's internal
|
||||
``ErrorResponse`` shape (``{"error": {"message": ...}}``). That
|
||||
shape doesn't match the ``CohereError`` schema advertised on the
|
||||
route's OpenAPI ``responses``, so clients (and schema-conformance
|
||||
tests like ``test_openai_schema.py``) would see a mismatch on
|
||||
those paths. This middleware normalises any error body on
|
||||
``/cohere/*`` responses to :class:`CohereError`.
|
||||
"""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if not request.url.path.startswith(_COHERE_PATH_PREFIX):
|
||||
return response
|
||||
if response.status_code < 400:
|
||||
return response
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if not content_type.startswith("application/json"):
|
||||
return response
|
||||
|
||||
body = b"".join([chunk async for chunk in response.body_iterator])
|
||||
translated = _translate_vllm_error_body(body, request)
|
||||
if translated is not None:
|
||||
return translated
|
||||
passthrough_headers = {
|
||||
k: v
|
||||
for k, v in response.headers.items()
|
||||
if k.lower() != "content-length"
|
||||
}
|
||||
return Response(
|
||||
content=body,
|
||||
status_code=response.status_code,
|
||||
headers=passthrough_headers,
|
||||
media_type=content_type,
|
||||
)
|
||||
|
||||
def _translate_vllm_error_body(raw: bytes, request: Request) -> JSONResponse | None:
|
||||
"""Translate a vLLM ``ErrorResponse`` body to a ``CohereError`` body.
|
||||
|
||||
Returns ``None`` if ``raw`` does not match the vLLM error envelope
|
||||
(which signals the middleware to pass the body through unchanged).
|
||||
"""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
return None
|
||||
if not (
|
||||
isinstance(data, dict)
|
||||
and isinstance(data.get("error"), dict)
|
||||
and "message" in data["error"]
|
||||
):
|
||||
return None
|
||||
try:
|
||||
err = ErrorResponse.model_validate(data)
|
||||
except Exception: # noqa: BLE001 - malformed envelope; pass through
|
||||
return None
|
||||
return _error_response(err, request)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Register the enabled ``/cohere/v2/chat`` routes on ``app``.
|
||||
|
||||
No-op when either:
|
||||
|
||||
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
|
||||
Cohere v2 endpoints are opt-in because they carry Cohere-specific
|
||||
request/response semantics (grounding citations, tool_plan,
|
||||
PLAN/THINKING_CONTENT blocks) that are only meaningful when
|
||||
serving a Cohere Command-family model.
|
||||
* the optional ``cohere`` SDK isn't installed (the v2 protocol
|
||||
models live there)
|
||||
|
||||
The two skip paths log at different levels: an operator who set
|
||||
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
|
||||
a WARNING (they explicitly asked for the endpoints and they're
|
||||
silently absent), whereas the default-off skip logs at debug.
|
||||
"""
|
||||
enabled = envs.VLLM_ENABLE_COHERE_API
|
||||
if not enabled:
|
||||
logger.debug(
|
||||
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints "
|
||||
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them."
|
||||
)
|
||||
return
|
||||
if not _SDK_AVAILABLE:
|
||||
logger.warning(
|
||||
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
|
||||
"installed; /cohere/v2/chat will not be exposed. Install "
|
||||
"with `pip install cohere` to enable the endpoints."
|
||||
)
|
||||
return
|
||||
app.include_router(router)
|
||||
args = getattr(app.state, "args", None)
|
||||
if getattr(args, "enable_scale_out", False) or getattr(args, "tokens_only", False):
|
||||
app.include_router(render_router)
|
||||
app.add_middleware(CohereErrorEnvelopeMiddleware)
|
||||
@@ -1,144 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM chat-protocol extensions for grounded (citation-carrying) models.
|
||||
|
||||
This module keeps the OpenAI chat completion protocol classes
|
||||
(:class:`ChatMessage` / :class:`DeltaMessage`) free of Cohere-specific fields.
|
||||
It provides two things:
|
||||
|
||||
* :class:`Citation` / :class:`CitationSource`: the vLLM-internal citation
|
||||
representation produced by the Cohere parser
|
||||
(:mod:`vllm.parser.cohere_command`) and consumed by
|
||||
:mod:`vllm.entrypoints.cohere.serving`. This is *not* the on-wire Cohere
|
||||
SDK shape -- that conversion happens in the serving layer.
|
||||
* :class:`CohereChatMessage` / :class:`CohereDeltaMessage`: :class:`ChatMessage`
|
||||
/ :class:`DeltaMessage` subclasses that add a ``citations`` field. They
|
||||
are only instantiated by the citation-aware serving handler
|
||||
(:class:`vllm.entrypoints.cohere.serving.CohereServingChatV2`) and by the
|
||||
Cohere reasoning parser's streaming path. Non-Cohere code paths continue
|
||||
to construct plain :class:`ChatMessage` / :class:`DeltaMessage`.
|
||||
|
||||
The module intentionally does *not* import the ``cohere`` Python SDK so the
|
||||
reasoning parser (which is Cohere-model-specific but ships without the SDK
|
||||
dependency) can import from here freely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_serializer
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
|
||||
|
||||
class CitationSource(OpenAIBaseModel):
|
||||
"""Source attribution for a :class:`Citation`.
|
||||
|
||||
Mirrors the shape used by Cohere's Chat v2 API. ``type`` is the source
|
||||
discriminator (``document`` or ``tool``); ``id`` is the citing
|
||||
document/tool-output identifier; ``document`` and ``tool_output`` carry
|
||||
the original payload that produced the citation.
|
||||
|
||||
Sources are fully resolved by the parser at emit time
|
||||
(see :func:`_melody_sources_to_vllm` in
|
||||
:mod:`vllm.parser.cohere_command`) using the
|
||||
``POSITION_TO_SOURCE_KEY`` map forwarded through
|
||||
``chat_template_kwargs`` by
|
||||
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._apply_cohere_template_kwargs`.
|
||||
That means every instance that reaches the serving / wire layer
|
||||
already has ``type`` / ``id`` populated; the serving layer only has
|
||||
to coerce to :class:`cohere.types.Citation` and rewrite
|
||||
``THINKING_CONTENT`` -> ``PLAN`` for non-reasoning models.
|
||||
"""
|
||||
|
||||
type: Literal["document", "tool"] | None = None
|
||||
id: str | None = None
|
||||
document: dict[str, Any] | None = None
|
||||
tool_output: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class Citation(OpenAIBaseModel):
|
||||
"""A citation grounding a span of generated text in source material.
|
||||
|
||||
vLLM-internal representation used by the Cohere reasoning parser and
|
||||
the Cohere v2 serving layer. This is not the on-wire Cohere SDK shape:
|
||||
conversion to the SDK's ``cohere.types.Citation`` happens in
|
||||
:mod:`vllm.entrypoints.cohere.serving`.
|
||||
"""
|
||||
|
||||
start: int | None = None
|
||||
"""Start character offset in the surrounding text content."""
|
||||
end: int | None = None
|
||||
"""End character offset (exclusive) in the surrounding text content."""
|
||||
text: str | None = None
|
||||
"""The cited text snippet."""
|
||||
sources: list[CitationSource] = Field(default_factory=list)
|
||||
"""Source documents / tool outputs that ground this citation."""
|
||||
content_index: int | None = None
|
||||
"""Index of the content block this citation refers to (when the message
|
||||
has multiple content blocks)."""
|
||||
type: Literal["TEXT_CONTENT", "THINKING_CONTENT", "PLAN"] | None = None
|
||||
"""Which kind of content block this citation grounds: the user-visible
|
||||
text (``TEXT_CONTENT``), a thinking block (``THINKING_CONTENT``), or a
|
||||
tool-plan block (``PLAN``). ``None`` means unspecified."""
|
||||
|
||||
|
||||
class CohereChatMessage(ChatMessage):
|
||||
""":class:`ChatMessage` extension carrying grounding citations.
|
||||
|
||||
Only instantiated by :class:`CohereServingChatV2` (and any other
|
||||
:class:`OpenAIServingChat` subclass that opts in by overriding
|
||||
``_create_chat_message``). Regular OpenAI-compatible handlers keep
|
||||
emitting plain :class:`ChatMessage` so their response schema is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares ``message: SerializeAsAny[ChatMessage]``,
|
||||
so pydantic serializes this subclass with its own schema (including
|
||||
``citations``) when it flows through
|
||||
:class:`ChatCompletionResponseChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# ``mode="wrap"`` fully overrides (rather than chains) the
|
||||
# parent's ``@model_serializer``, so we explicitly delegate via
|
||||
# ``super()._serialize(handler)`` to preserve the parent's
|
||||
# cleanup (e.g. stripping empty ``tool_calls``). Then we drop an
|
||||
# unset ``citations`` field so the wire matches the OpenAI-style
|
||||
# contract that optional vLLM extensions are omitted rather than
|
||||
# serialized as ``null``.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
|
||||
|
||||
class CohereDeltaMessage(DeltaMessage):
|
||||
""":class:`DeltaMessage` extension carrying grounding citations for streaming.
|
||||
|
||||
Emitted by :class:`vllm.parser.cohere_command.CohereCommandParser`
|
||||
on delta events whose payload includes citations. Non-Cohere parsers
|
||||
return plain :class:`DeltaMessage`, so their streamed shape is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares
|
||||
``delta: SerializeAsAny[DeltaMessage]``, so pydantic serializes this
|
||||
subclass with its own schema when it flows through
|
||||
:class:`ChatCompletionResponseStreamChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# See ``CohereChatMessage._serialize`` for why we delegate via
|
||||
# ``super()`` instead of calling ``handler(self)`` directly.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
@@ -1,389 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cohere Chat v2 API protocol.
|
||||
|
||||
The bulk of the wire types come straight from the official ``cohere``
|
||||
Python SDK so we stay in sync with the upstream specification and
|
||||
avoid re-declaring the schema. We only own three things locally:
|
||||
|
||||
1. The top-level request body model (the SDK doesn't ship one — its
|
||||
``ClientV2.chat`` takes the body as kwargs), with vLLM-specific
|
||||
extensions (``kv_transfer_params`` / ``chat_template_kwargs``).
|
||||
2. The non-streaming response envelope (the SDK exposes the message
|
||||
shape via :class:`AssistantMessageResponse` but no full response
|
||||
wrapper).
|
||||
3. The streaming discriminated union (the SDK exports each event type
|
||||
individually but not as a combined ``Annotated[Union[...],
|
||||
discriminator]``).
|
||||
|
||||
Importing this module pulls in the ``cohere`` package. The router that
|
||||
mounts ``POST /cohere/v2/chat`` guards on that import succeeding so vLLM still
|
||||
boots without the SDK installed.
|
||||
|
||||
See https://docs.cohere.com/reference/chat for the upstream spec.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from cohere import types as _sdk
|
||||
from cohere.types import (
|
||||
AssistantChatMessageV2,
|
||||
AssistantMessageResponse,
|
||||
ChatMessageV2,
|
||||
ChatRequestSafetyMode,
|
||||
Citation,
|
||||
CitationOptions,
|
||||
Document,
|
||||
ResponseFormatV2,
|
||||
SystemChatMessageV2,
|
||||
Thinking,
|
||||
ToolCallV2,
|
||||
ToolChatMessageV2,
|
||||
ToolV2,
|
||||
UserChatMessageV2,
|
||||
)
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
# Re-export the SDK wire-format types alongside our local extensions so
|
||||
# ``vllm.entrypoints.cohere.serving`` and friends can import everything
|
||||
# they need from this module.
|
||||
__all__ = [
|
||||
"AssistantChatMessageV2",
|
||||
"AssistantMessageResponse",
|
||||
"ChatMessageV2",
|
||||
"Citation",
|
||||
"CitationEndEvent",
|
||||
"CitationOptions",
|
||||
"CitationStartEvent",
|
||||
"CohereChatV2Request",
|
||||
"CohereChatV2Response",
|
||||
"CohereError",
|
||||
"CohereFinishReason",
|
||||
"CohereLogprobItem",
|
||||
"CohereUsage",
|
||||
"CohereUsageBilledUnits",
|
||||
"CohereUsageTokens",
|
||||
"ContentDeltaEvent",
|
||||
"ContentEndEvent",
|
||||
"ContentStartEvent",
|
||||
"Document",
|
||||
"MessageEndEvent",
|
||||
"MessageStartEvent",
|
||||
"ResponseFormatV2",
|
||||
"SystemChatMessageV2",
|
||||
"Thinking",
|
||||
"ToolCallDeltaEvent",
|
||||
"ToolCallEndEvent",
|
||||
"ToolCallStartEvent",
|
||||
"ToolCallV2",
|
||||
"ToolChatMessageV2",
|
||||
"ToolPlanDeltaEvent",
|
||||
"ToolV2",
|
||||
"UserChatMessageV2",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Errors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereError(BaseModel):
|
||||
"""Top-level error body returned by ``/cohere/v2/chat`` error responses.
|
||||
|
||||
Cohere's documented error schemas are uniform: ``{message, id}``.
|
||||
"""
|
||||
|
||||
message: str
|
||||
id: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool choice / finish reasons
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# These literals aren't first-class enums in the SDK but are documented at
|
||||
# https://docs.cohere.com/reference/chat. We declare them here so the
|
||||
# request/response models can validate them.
|
||||
|
||||
CohereToolChoice = Literal["REQUIRED", "NONE"]
|
||||
|
||||
CohereFinishReason = Literal[
|
||||
"COMPLETE",
|
||||
"STOP_SEQUENCE",
|
||||
"MAX_TOKENS",
|
||||
"TOOL_CALL",
|
||||
"ERROR",
|
||||
"TIMEOUT",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Request(BaseModel):
|
||||
"""Cohere Chat v2 request body.
|
||||
|
||||
Mirrors the schema documented at https://docs.cohere.com/reference/chat.
|
||||
All structured fields delegate to the official SDK types so the body
|
||||
schema stays in sync with the upstream spec.
|
||||
"""
|
||||
|
||||
model: str
|
||||
messages: list[ChatMessageV2]
|
||||
stream: bool | None = False
|
||||
|
||||
# Tooling
|
||||
tools: list[ToolV2] | None = None
|
||||
strict_tools: bool | None = None
|
||||
tool_choice: CohereToolChoice | None = None
|
||||
|
||||
# Grounding
|
||||
documents: list[str | Document] | None = None
|
||||
citation_options: CitationOptions | None = None
|
||||
|
||||
# Output
|
||||
response_format: ResponseFormatV2 | None = None
|
||||
safety_mode: ChatRequestSafetyMode | None = None
|
||||
max_tokens: int | None = None
|
||||
stop_sequences: (
|
||||
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||
) = None
|
||||
|
||||
# Sampling
|
||||
temperature: float | None = None
|
||||
seed: int | None = None
|
||||
frequency_penalty: float | None = None
|
||||
presence_penalty: float | None = None
|
||||
k: int | None = None
|
||||
p: float | None = None
|
||||
logprobs: bool | None = None
|
||||
|
||||
# Reasoning
|
||||
thinking: Thinking | None = None
|
||||
|
||||
# Scheduling
|
||||
priority: int | None = Field(default=None, ge=-(2**63), le=2**63 - 1)
|
||||
|
||||
# vLLM-specific extensions (not in Cohere spec). These mirror what the
|
||||
# Anthropic and OpenAI surfaces already expose so V2 callers can reach
|
||||
# the same engine knobs when needed.
|
||||
watermarking: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Whether to apply the engine's configured watermark to this request."
|
||||
),
|
||||
)
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the template."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def _validate_model(cls, v: str) -> str:
|
||||
if not v:
|
||||
raise VLLMValidationError("model is required", parameter="model")
|
||||
return v
|
||||
|
||||
@field_validator("max_tokens")
|
||||
@classmethod
|
||||
def _validate_max_tokens(cls, v: int | None) -> int | None:
|
||||
if v is not None and v < 0:
|
||||
raise VLLMValidationError(
|
||||
"max_tokens must be non-negative",
|
||||
parameter="max_tokens",
|
||||
value=v,
|
||||
)
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_tool_choice(self) -> CohereChatV2Request:
|
||||
if self.tool_choice == "REQUIRED" and not self.tools:
|
||||
raise VLLMValidationError(
|
||||
'tool_choice="REQUIRED" requires at least one tool in tools.',
|
||||
parameter="tool_choice",
|
||||
value=self.tool_choice,
|
||||
)
|
||||
return self
|
||||
|
||||
@field_validator("messages", mode="before")
|
||||
@classmethod
|
||||
def _normalize_message_roles(cls, v: Any) -> Any:
|
||||
"""Rewrite OpenAI-style ``developer`` roles to ``system``.
|
||||
|
||||
Cohere's v2 ``ChatMessageV2`` discriminated union only admits
|
||||
the four literal roles ``user`` / ``assistant`` / ``system`` /
|
||||
``tool``. OpenAI's ``developer`` role is documented as
|
||||
high-priority system instructions, so we alias it onto
|
||||
``system`` *before* the SDK discriminator runs (otherwise it
|
||||
rejects the message with a ``literal_error`` against each union
|
||||
member). Mirrors ``_role_to_melody`` in the renderer so the
|
||||
same alias is honoured no matter which surface the message
|
||||
arrives through.
|
||||
|
||||
``mode="before"`` is required so the rewrite happens before the
|
||||
``list[ChatMessageV2]`` coercion runs the SDK's discriminated
|
||||
union; a default-mode validator would never see ``developer``
|
||||
because validation would have already failed. On any
|
||||
structural malformation (non-iterable input, items without a
|
||||
dict-shaped ``role`` field, etc.) we hand ``v`` back unchanged
|
||||
and let Pydantic's normal coercion surface a precise error.
|
||||
"""
|
||||
try:
|
||||
return [
|
||||
{**msg, "role": "system"}
|
||||
if msg.get("role", "").lower() == "developer"
|
||||
else msg
|
||||
for msg in v
|
||||
]
|
||||
except (AttributeError, TypeError):
|
||||
return v
|
||||
|
||||
@field_validator("messages")
|
||||
@classmethod
|
||||
def _validate_messages(cls, v: list[ChatMessageV2]) -> list[ChatMessageV2]:
|
||||
if not v:
|
||||
raise VLLMValidationError(
|
||||
"messages must contain at least one message", parameter="messages"
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usage / Logprobs
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The Cohere SDK only exposes a v1 ``ApiMetaBilledUnits``. The v2 usage
|
||||
# envelope is documented separately in the OpenAPI spec and we declare it
|
||||
# here.
|
||||
|
||||
|
||||
class CohereUsageBilledUnits(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
search_units: float | None = None
|
||||
classifications: float | None = None
|
||||
|
||||
|
||||
class CohereUsageTokens(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereUsage(BaseModel):
|
||||
billed_units: CohereUsageBilledUnits | None = None
|
||||
tokens: CohereUsageTokens | None = None
|
||||
cached_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereLogprobItem(BaseModel):
|
||||
text: str | None = None
|
||||
token_ids: list[int]
|
||||
logprobs: list[float] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-streaming response
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Response(BaseModel):
|
||||
"""Cohere Chat v2 non-streaming response body.
|
||||
|
||||
Wraps the SDK :class:`AssistantMessageResponse` (the message shape) in
|
||||
the documented v2 response envelope (``id``, ``finish_reason``,
|
||||
``usage``, ``logprobs``). The single constructor in
|
||||
:class:`CohereServingChatV2._chat_completion_to_v2` is responsible for
|
||||
supplying a non-empty ``id`` (falling back to a synthesized one if
|
||||
the upstream response is missing it) to this model.
|
||||
"""
|
||||
|
||||
id: str
|
||||
finish_reason: CohereFinishReason
|
||||
message: AssistantMessageResponse
|
||||
usage: CohereUsage | None = None
|
||||
logprobs: list[CohereLogprobItem] | None = None
|
||||
|
||||
# vLLM-specific extension.
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming events
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Cohere V2 streams a sequence of typed JSON events delivered as Server-
|
||||
# Sent Events; each event's ``type`` field carries the discriminator. The
|
||||
# SDK exposes a Pydantic model per event but none of them declare ``type``
|
||||
# as a field (the SDK relies on its own deserializer for discrimination),
|
||||
# so a naive ``model_dump_json()`` would silently drop the discriminator
|
||||
# and break clients that demux on ``type``.
|
||||
#
|
||||
# We therefore subclass each SDK event and bake the wire-format ``type``
|
||||
# string in as a ``Literal`` field with a default. ``model_dump()`` now
|
||||
# emits ``type`` for free, and surfaces can simply construct the event
|
||||
# class and serialize it -- no manual ``type`` parameter needed.
|
||||
#
|
||||
# See https://docs.cohere.com/v2/docs/streaming and the OpenAPI
|
||||
# ``StreamedChatResponseV2`` schema for the wire-format reference.
|
||||
|
||||
|
||||
class MessageStartEvent(_sdk.ChatMessageStartEvent):
|
||||
type: Literal["message-start"] = "message-start"
|
||||
|
||||
|
||||
class ContentStartEvent(_sdk.ChatContentStartEvent):
|
||||
type: Literal["content-start"] = "content-start"
|
||||
|
||||
|
||||
class ContentDeltaEvent(_sdk.ChatContentDeltaEvent):
|
||||
type: Literal["content-delta"] = "content-delta"
|
||||
|
||||
|
||||
class ContentEndEvent(_sdk.ChatContentEndEvent):
|
||||
type: Literal["content-end"] = "content-end"
|
||||
|
||||
|
||||
class ToolPlanDeltaEvent(_sdk.ChatToolPlanDeltaEvent):
|
||||
type: Literal["tool-plan-delta"] = "tool-plan-delta"
|
||||
|
||||
|
||||
class ToolCallStartEvent(_sdk.ChatToolCallStartEvent):
|
||||
type: Literal["tool-call-start"] = "tool-call-start"
|
||||
|
||||
|
||||
class ToolCallDeltaEvent(_sdk.ChatToolCallDeltaEvent):
|
||||
type: Literal["tool-call-delta"] = "tool-call-delta"
|
||||
|
||||
|
||||
class ToolCallEndEvent(_sdk.ChatToolCallEndEvent):
|
||||
type: Literal["tool-call-end"] = "tool-call-end"
|
||||
|
||||
|
||||
class CitationStartEvent(_sdk.CitationStartEvent):
|
||||
type: Literal["citation-start"] = "citation-start"
|
||||
|
||||
|
||||
class CitationEndEvent(_sdk.CitationEndEvent):
|
||||
type: Literal["citation-end"] = "citation-end"
|
||||
|
||||
|
||||
class MessageEndEvent(_sdk.ChatMessageEndEvent):
|
||||
type: Literal["message-end"] = "message-end"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -31,7 +31,6 @@ from vllm.renderers.inputs.preprocess import (
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.utils.counter import Counter
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from vllm.utils.tqdm_utils import maybe_tqdm
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
@@ -191,10 +190,7 @@ class OfflineInferenceMixin:
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
continue_final_message=continue_final_message,
|
||||
tools=tools,
|
||||
tokenize=(
|
||||
is_mistral_tokenizer(renderer.tokenizer)
|
||||
or self.model_config.enable_prompt_embeds
|
||||
),
|
||||
tokenize=self.model_config.enable_prompt_embeds,
|
||||
),
|
||||
),
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
|
||||
@@ -4,8 +4,6 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
@@ -37,17 +35,6 @@ def register_generate_api_routers(app: FastAPI):
|
||||
|
||||
register_completion_api_router(app)
|
||||
|
||||
from vllm.entrypoints.anthropic.api_router import (
|
||||
attach_router as register_anthropic_api_router,
|
||||
)
|
||||
|
||||
register_anthropic_api_router(app)
|
||||
|
||||
from vllm.entrypoints.cohere.api_router import (
|
||||
attach_router as register_cohere_api_router,
|
||||
)
|
||||
|
||||
register_cohere_api_router(app)
|
||||
|
||||
from .generative_scoring.api_router import register_generative_scoring_api_router
|
||||
|
||||
@@ -62,24 +49,6 @@ async def init_generate_state(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
|
||||
# The Cohere serving handler depends on the optional `cohere` SDK for
|
||||
# its wire-format protocol models, and is additionally gated on the
|
||||
# `VLLM_ENABLE_COHERE_API` env flag (see
|
||||
# `vllm.entrypoints.cohere.api_router.attach_router`). Skip the import
|
||||
# entirely when the endpoint isn't going to be exposed, both because
|
||||
# the SDK may not be installed and because the serving object holds
|
||||
# nontrivial state (chat handler, warmup) that would otherwise be
|
||||
# unused.
|
||||
if envs.VLLM_ENABLE_COHERE_API:
|
||||
try:
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
except ImportError:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
else:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
OpenAIServingChatBatch,
|
||||
)
|
||||
@@ -87,6 +56,7 @@ async def init_generate_state(
|
||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||
from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
|
||||
# Applied before any serving class is constructed so that each one picks
|
||||
# up the chosen mode on its first cache miss.
|
||||
@@ -168,54 +138,6 @@ async def init_generate_state(
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
state.anthropic_serving_messages = (
|
||||
AnthropicServingMessages(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
disabled_thinking_effort=args.anthropic_disabled_thinking_effort,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
state.cohere_serving_chat_v2 = (
|
||||
CohereServingChatV2(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
is_reasoning_model=args.cohere_is_reasoning_model,
|
||||
)
|
||||
if CohereServingChatV2 is not None and "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
|
||||
from .generative_scoring.serving import ServingGenerativeScoring
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
EndpointFn,
|
||||
GetHandlerFn,
|
||||
RequestType,
|
||||
)
|
||||
|
||||
|
||||
def get_generate_invocation_types(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
# NOTE: Items defined earlier take higher priority
|
||||
invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.openai.chat_completion.api_router import (
|
||||
chat,
|
||||
create_chat_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.api_router import (
|
||||
completion,
|
||||
create_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
|
||||
invocation_types += [
|
||||
(ChatCompletionRequest, (chat, create_chat_completion)),
|
||||
(CompletionRequest, (completion, create_completion)),
|
||||
]
|
||||
|
||||
return invocation_types
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,86 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.decisions.state import init_decisions_state
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
from .protocol import StructuredDecisionRequest
|
||||
from .serving import ServingStructuredDecisions
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def structured_decisions(request: Request) -> ServingStructuredDecisions | None:
|
||||
return getattr(request.app.state, "serving_structured_decisions", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/systemone",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_structured_decision(
|
||||
request: StructuredDecisionRequest, raw_request: Request
|
||||
):
|
||||
handler = structured_decisions(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support the structured decision API"
|
||||
)
|
||||
result = await handler.create_decision(request, raw_request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def register_structured_decisions_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
def init_structured_decisions_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
strategy = init_decisions_state(
|
||||
engine_client,
|
||||
state,
|
||||
args,
|
||||
request_logger,
|
||||
load_chat_template(args.chat_template),
|
||||
default_chat_template_kwargs,
|
||||
supported_tasks,
|
||||
)
|
||||
state.serving_structured_decisions = (
|
||||
ServingStructuredDecisions(
|
||||
state.openai_serving_models, strategy, request_logger=request_logger
|
||||
)
|
||||
if strategy is not None
|
||||
else None
|
||||
)
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Request and response models of the structured decisions API (/v1/systemone)."""
|
||||
|
||||
import time
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
|
||||
class QuestionSpec(OpenAIBaseModel):
|
||||
type: str
|
||||
instructions: Any = ""
|
||||
criteria: Any = None
|
||||
|
||||
|
||||
class StructuredDecisionRequest(OpenAIBaseModel):
|
||||
model: str | None = None
|
||||
state: Any = Field(..., description="What the questions are about: text or JSON.")
|
||||
questions: dict[str, QuestionSpec] = Field(
|
||||
..., description="Question id to question, asked in this order."
|
||||
)
|
||||
instructions: str | None = Field(
|
||||
default=None, description="Context placed ahead of the questions."
|
||||
)
|
||||
images: list[str] | None = Field(
|
||||
default=None, description="Image URLs or data URLs, for models that see."
|
||||
)
|
||||
audio: list[str] | None = Field(
|
||||
default=None, description="Audio URLs or data URLs, for models that hear."
|
||||
)
|
||||
videos: list[str] | None = Field(
|
||||
default=None, description="Video URLs or data URLs, for models that see."
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
|
||||
request_id: str = Field(default_factory=random_uuid)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
|
||||
class DecisionUsage(OpenAIBaseModel):
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
||||
class QuestionDiagnostics(OpenAIBaseModel):
|
||||
label_mass: float = Field(
|
||||
description="Probability the model put on the labels, over the full "
|
||||
"vocabulary. A low value means most of the probability went to tokens "
|
||||
"that are not labels."
|
||||
)
|
||||
argmax_is_label: bool
|
||||
|
||||
|
||||
class StructuredDecisionResponse(OpenAIBaseModel):
|
||||
id: str
|
||||
object: Literal["structured_decision"] = "structured_decision"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
answers: dict[str, dict[str, Any]]
|
||||
usage: DecisionUsage
|
||||
diagnostics: dict[str, QuestionDiagnostics]
|
||||
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
build_question,
|
||||
)
|
||||
from vllm.entrypoints.openai.decisions.serving import BaseServingDecisions
|
||||
from vllm.entrypoints.openai.decisions.strategies import DecisionLimits
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
|
||||
from .protocol import (
|
||||
DecisionUsage,
|
||||
QuestionDiagnostics,
|
||||
StructuredDecisionRequest,
|
||||
StructuredDecisionResponse,
|
||||
)
|
||||
|
||||
|
||||
def parse_questions(
|
||||
request: StructuredDecisionRequest,
|
||||
limits: DecisionLimits,
|
||||
) -> list[Question]:
|
||||
if not request.questions:
|
||||
raise StructuredDecisionError("questions: needs at least one question")
|
||||
if len(request.questions) > limits.max_questions:
|
||||
raise StructuredDecisionError(
|
||||
f"questions: at most {limits.max_questions} for this model"
|
||||
)
|
||||
questions = []
|
||||
for qid, spec in request.questions.items():
|
||||
if spec.model_extra:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: unknown field(s) {sorted(spec.model_extra)}"
|
||||
)
|
||||
questions.append(
|
||||
build_question(
|
||||
qid,
|
||||
spec.type,
|
||||
spec.instructions,
|
||||
spec.criteria,
|
||||
limits.max_options,
|
||||
labeled=limits.labeled,
|
||||
)
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
class ServingStructuredDecisions(BaseServingDecisions):
|
||||
def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]:
|
||||
urls = {
|
||||
"image": request.images,
|
||||
"audio": request.audio,
|
||||
"video": request.videos,
|
||||
}
|
||||
media = {modality: items for modality, items in urls.items() if items}
|
||||
if unsupported := media.keys() - self.strategy.modalities:
|
||||
names = ", ".join(sorted(unsupported))
|
||||
raise StructuredDecisionError(f"this model does not take {names} inputs")
|
||||
return media
|
||||
|
||||
async def create_decision(
|
||||
self,
|
||||
request: StructuredDecisionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> StructuredDecisionResponse | ErrorResponse:
|
||||
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
|
||||
return error
|
||||
engine_client = self.strategy.context.engine_client
|
||||
if engine_client.errored:
|
||||
raise engine_client.dead_error
|
||||
|
||||
base_id = self._base_request_id(raw_request, default=request.request_id)
|
||||
request_id = f"decision-{base_id}"
|
||||
try:
|
||||
questions = parse_questions(request, self.limits)
|
||||
media = self._media(request)
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine_client.check_admission(self.strategy.num_requests(questions))
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
request.instructions,
|
||||
self.strategy.render_state(request.state),
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
cache_salt=request.cache_salt,
|
||||
trace_headers=trace_headers,
|
||||
media=media,
|
||||
)
|
||||
except StructuredDecisionError as e:
|
||||
return self.create_error_response(e)
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
|
||||
answers: dict[str, dict[str, Any]] = {}
|
||||
diagnostics: dict[str, QuestionDiagnostics] = {}
|
||||
for q, read in zip(questions, reads):
|
||||
answers[q.id] = q.type.answer(q, read.probs, read.label_mass)
|
||||
diagnostics[q.id] = QuestionDiagnostics(
|
||||
label_mass=read.label_mass, argmax_is_label=read.argmax_is_label
|
||||
)
|
||||
return StructuredDecisionResponse(
|
||||
id=request_id,
|
||||
model=self.models.model_name(lora_request),
|
||||
answers=answers,
|
||||
usage=DecisionUsage(
|
||||
input_tokens=sum(r.input_tokens for r in reads),
|
||||
output_tokens=sum(r.output_tokens for r in reads),
|
||||
),
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
@@ -7,7 +7,6 @@ from typing import cast
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
@@ -16,7 +15,6 @@ from vllm.entrypoints.serve.utils.api_utils import process_lora_modules
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||
|
||||
@@ -65,7 +63,7 @@ async def init_app_state(
|
||||
state.vllm_config = vllm_config
|
||||
state.args = args
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
state.tool_server = (
|
||||
await init_tool_server(args) if "generate" in supported_tasks else None
|
||||
)
|
||||
@@ -103,22 +101,6 @@ async def init_app_state(
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
@@ -152,19 +134,6 @@ async def init_app_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
init_structured_decisions_state,
|
||||
)
|
||||
|
||||
init_structured_decisions_state(
|
||||
engine_client,
|
||||
state,
|
||||
args,
|
||||
request_logger,
|
||||
supported_tasks,
|
||||
default_chat_template_kwargs,
|
||||
)
|
||||
|
||||
await init_endpoint_plugins_state(engine_client, state, args)
|
||||
|
||||
state.enable_server_load_tracking = args.enable_server_load_tracking
|
||||
|
||||
@@ -25,33 +25,11 @@ def register_api_routers(
|
||||
|
||||
register_models_api_router(app)
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
register_structured_decisions_api_router,
|
||||
)
|
||||
|
||||
register_structured_decisions_api_router(app)
|
||||
|
||||
from vllm.entrypoints.openai.decisions.api_router import (
|
||||
register_decisions_api_router,
|
||||
)
|
||||
|
||||
register_decisions_api_router(app)
|
||||
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
attach_router as register_sagemaker_api_router,
|
||||
)
|
||||
|
||||
register_sagemaker_api_router(app, supported_tasks, model_config)
|
||||
|
||||
if envs.VLLM_SERVER_DEV_MODE:
|
||||
from vllm.entrypoints.serve import register_vllm_dev_api_routers
|
||||
|
||||
register_vllm_dev_api_routers(app)
|
||||
|
||||
from vllm.entrypoints.rl.factories import register_rl_api_routers
|
||||
|
||||
register_rl_api_routers(app)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.generate.api_router import (
|
||||
register_generate_api_routers,
|
||||
@@ -59,13 +37,6 @@ def register_api_routers(
|
||||
|
||||
register_generate_api_routers(app)
|
||||
|
||||
from vllm.entrypoints.serve.elastic_ep.api_router import (
|
||||
attach_router as elastic_ep_attach_router,
|
||||
)
|
||||
|
||||
elastic_ep_attach_router(app)
|
||||
|
||||
if "generate" in supported_tasks or "render" in supported_tasks:
|
||||
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
|
||||
|
||||
register_scale_out_api_routers(app, supported_tasks)
|
||||
|
||||
@@ -8,7 +8,6 @@ from fastapi import FastAPI
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||
|
||||
@@ -35,22 +34,16 @@ def build_app(
|
||||
app = FastAPI(
|
||||
openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
|
||||
)
|
||||
elif args.enable_offline_docs:
|
||||
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
|
||||
else:
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
|
||||
app.state.args = args
|
||||
app.root_path = args.root_path
|
||||
|
||||
register_api_routers(args, app, supported_tasks, model_config)
|
||||
|
||||
# Endpoint plugin hooks run after core routers so plugins can inspect them.
|
||||
# This runs even for the CPU only render server. A plugin eligible for
|
||||
# the `render` task still gets its routes registered. It receives
|
||||
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
|
||||
attach_endpoint_plugins(app, supported_tasks)
|
||||
|
||||
init_exception_handler(app)
|
||||
init_entrypoints_middleware(args, app, supported_tasks)
|
||||
app = sagemaker_standards_bootstrap(app)
|
||||
return app
|
||||
|
||||
@@ -14,9 +14,6 @@ from typing import Any, Literal
|
||||
import vllm.envs as envs
|
||||
from vllm.config import config
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
|
||||
from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicDisabledThinkingEffortOption,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.chat_utils import (
|
||||
@@ -172,32 +169,6 @@ class BaseFrontendArgs:
|
||||
"""If set to False, output deltas will not be logged. Relevant only if
|
||||
--enable-log-outputs is set.
|
||||
"""
|
||||
cohere_is_reasoning_model: bool = True
|
||||
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
|
||||
reasoning Command-family model. When True (default), the assistant's
|
||||
chain-of-thought is surfaced as a ``thinking`` content block (or
|
||||
``content-*`` events on the stream). When False, reasoning is
|
||||
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
|
||||
events) whenever the model emits tool calls, matching older non-
|
||||
reasoning Command models. Has no effect on the non-Cohere
|
||||
endpoints."""
|
||||
cohere_format: str = "cmd4"
|
||||
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
|
||||
format to render: ``cmd4`` (current Command A+ models; default) or
|
||||
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
|
||||
wrong format silently produces a prompt the model wasn't trained
|
||||
on, which most commonly manifests as the model emitting text but no
|
||||
citations / tool calls / thinking blocks. Equivalent to passing
|
||||
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
|
||||
explicit request-level ``chat_template_kwargs.cohere_format`` takes
|
||||
priority."""
|
||||
anthropic_disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto"
|
||||
"""Anthropic ``/v1/messages`` only. The ``reasoning_effort`` used for
|
||||
requests with ``thinking: {"type": "disabled"}``. ``none`` turns thinking
|
||||
off for models that support it; ``low`` suits models that always think
|
||||
(e.g. GLM-5.3) or reject ``none`` (e.g. gpt-oss). ``auto`` (default) uses
|
||||
``low`` when the renderer rejects ``none`` or renders it the same as a
|
||||
thinking effort, and ``none`` otherwise."""
|
||||
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
|
||||
"""If set to True, log the stack trace of error responses"""
|
||||
tokens_only: bool = False
|
||||
@@ -207,10 +178,10 @@ class BaseFrontendArgs:
|
||||
"""
|
||||
enable_scale_out: bool = False
|
||||
"""
|
||||
If set to True, register the scale-out endpoints (`/render`, `/derender`,
|
||||
`/inference/v1/generate` and `/inference/v1/abort_requests`) on `vllm serve`.
|
||||
Has no effect on `vllm launch render` or `vllm serve --tokens-only` which
|
||||
always register their required endpoints regardless of this flag.
|
||||
If set to True, register the scale-out endpoints (`/inference/v1/generate`
|
||||
and `/inference/v1/abort_requests`) on `vllm serve`. `vllm serve
|
||||
--tokens-only` always registers the endpoints it needs regardless of this
|
||||
flag.
|
||||
"""
|
||||
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
|
||||
"""Controls the ``system_fingerprint`` field on responses.
|
||||
@@ -360,11 +331,6 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
|
||||
"""Maximum number of HTTP headers allowed in a request for h11 parser.
|
||||
Helps mitigate header abuse. Default: 256."""
|
||||
enable_offline_docs: bool = False
|
||||
"""
|
||||
Enable offline FastAPI documentation for air-gapped environments.
|
||||
Uses vendored static assets bundled with vLLM.
|
||||
"""
|
||||
enable_flash_late_interaction: bool = True
|
||||
"""If set, run pooling score MaxSim on GPU in the API server process.
|
||||
Can significantly improve late-interaction scoring performance.
|
||||
@@ -404,16 +370,6 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
return frontend_kwargs
|
||||
|
||||
|
||||
def resolve_default_chat_template_kwargs(
|
||||
args: argparse.Namespace,
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
|
||||
kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
if getattr(args, "cohere_format", None):
|
||||
kwargs.setdefault("cohere_format", args.cohere_format)
|
||||
return kwargs
|
||||
|
||||
|
||||
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
"""Create the CLI argument parser used by the OpenAI API server.
|
||||
|
||||
@@ -453,9 +409,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Launch the Python gRPC (SMG VllmEngine) server on --port instead "
|
||||
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. "
|
||||
"Cannot be combined with --grpc-port, which enables the Rust "
|
||||
"frontend's gRPC services.",
|
||||
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc].",
|
||||
)
|
||||
parser = FrontendArgs.add_cli_args(parser)
|
||||
parser = AsyncEngineArgs.add_cli_args(parser)
|
||||
@@ -465,9 +419,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
|
||||
def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
"""Quick checks for model serve args that raise prior to loading."""
|
||||
# `vllm launch <component>` builds its parser with make_arg_parser too (see
|
||||
# LaunchSubcommandBase.add_cli_args), so its args are serve args as well.
|
||||
if hasattr(args, "subparser") and args.subparser not in ("serve", "launch"):
|
||||
if hasattr(args, "subparser") and args.subparser != "serve":
|
||||
return
|
||||
|
||||
# Ensure that the chat template is valid; raises if it likely isn't
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.scale_out.factories import init_render_state
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
|
||||
async def init_render_app_state(
|
||||
vllm_config: VllmConfig,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
) -> None:
|
||||
"""Initialise FastAPI app state for a CPU-only render server.
|
||||
|
||||
Unlike :func:`init_app_state` this function does not require an
|
||||
:class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
|
||||
preprocessing pipeline (renderer, input_processor)
|
||||
directly from the :class:`~vllm.config.VllmConfig`.
|
||||
"""
|
||||
served_model_names = args.served_model_name or [args.model]
|
||||
model_registry = OpenAIModelRegistry(
|
||||
model_config=vllm_config.model_config,
|
||||
base_model_paths=[
|
||||
BaseModelPath(name=name, model_path=args.model)
|
||||
for name in served_model_names
|
||||
],
|
||||
)
|
||||
|
||||
if args.enable_log_requests:
|
||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||
else:
|
||||
request_logger = None
|
||||
|
||||
renderer = renderer_from_config(vllm_config)
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||
state.tool_server = await init_tool_server(args)
|
||||
|
||||
# The config-resolved reasoning parser carries both the CLI flag (merged
|
||||
# in the entrypoint via `create_structured_outputs_config`) and any
|
||||
# model-specific default applied by `verify_and_update_config`
|
||||
# (e.g. "openai_gptoss" for gpt_oss), matching the main API server.
|
||||
# Keep `args.reasoning_parser` as the first source so callers that build
|
||||
# a VllmConfig without that merge still honor an explicit flag.
|
||||
reasoning_parser = (
|
||||
args.reasoning_parser or vllm_config.structured_outputs_config.reasoning_parser
|
||||
)
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.openai_serving_models = model_registry
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
model_registry,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
init_render_state(state, request_logger)
|
||||
|
||||
state.vllm_config = vllm_config
|
||||
# Disable stats logging — there is no engine to poll.
|
||||
state.log_stats = False
|
||||
state.engine_client = None
|
||||
state.args = args
|
||||
state.enable_server_load_tracking = False
|
||||
state.server_load_metrics = 0
|
||||
|
||||
# No `EngineClient` exists for the render server, so plugins get `None` and
|
||||
# must handle it themselves (see `EndpointPlugin.init_state`).
|
||||
await init_endpoint_plugins_state(None, state, args)
|
||||
@@ -1,129 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import asyncio
|
||||
import signal
|
||||
import socket
|
||||
from argparse import Namespace
|
||||
|
||||
from vllm import AsyncEngineArgs, envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import configure_logging_from_args, init_logger
|
||||
|
||||
from ..app import build_app
|
||||
from ..launcher import serve_http, setup_server
|
||||
from ..utils.server_utils import get_uvicorn_log_config
|
||||
from .app_state import init_render_app_state
|
||||
|
||||
logger = init_logger("vllm.entrypoints.launchers.render.entry")
|
||||
|
||||
|
||||
async def build_and_serve_renderer(
|
||||
vllm_config: VllmConfig,
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: Namespace,
|
||||
**uvicorn_kwargs,
|
||||
) -> asyncio.Task:
|
||||
"""Build FastAPI app for a CPU-only render server, initialize state, and
|
||||
start serving.
|
||||
|
||||
Returns the shutdown task for the caller to await.
|
||||
"""
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
app = build_app(args, ("render",))
|
||||
await init_render_app_state(vllm_config, app.state, args)
|
||||
|
||||
logger.info("Starting vLLM server on %s", listen_address)
|
||||
|
||||
return await serve_http(
|
||||
app,
|
||||
sock=sock,
|
||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||
# no access log will be output.
|
||||
access_log=not args.disable_uvicorn_access_log,
|
||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||
ssl_keyfile=args.ssl_keyfile,
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_ca_certs=args.ssl_ca_certs,
|
||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||
ssl_ciphers=args.ssl_ciphers,
|
||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||
h11_max_header_count=args.h11_max_header_count,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def run_launch_fastapi(args: argparse.Namespace) -> None:
|
||||
"""Run the online serving layer with FastAPI (no GPU inference)."""
|
||||
|
||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs
|
||||
# its own signal handlers. Once uvicorn is running it replaces this.
|
||||
def _interrupt_init(*_) -> None:
|
||||
raise KeyboardInterrupt("terminated")
|
||||
|
||||
signal.signal(signal.SIGTERM, _interrupt_init)
|
||||
|
||||
# 1. Socket binding
|
||||
listen_address, sock = setup_server(args, reuse_port=False)
|
||||
|
||||
# 2. Build and serve the API server
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
model_config = engine_args.create_model_config()
|
||||
|
||||
# Render servers preprocess data only — no inference, no quantized kernels.
|
||||
# Clear quantization so VllmConfig skips quant dtype/capability validation.
|
||||
model_config.quantization = None
|
||||
|
||||
# Render servers never allocate KV cache; suppress the spurious CPU KV
|
||||
# cache space warning from CpuPlatform.check_and_update_config.
|
||||
envs.VLLM_CPU_KVCACHE_SPACE = 0
|
||||
|
||||
# Merge frontend parser flags (e.g. --reasoning-parser) into the
|
||||
# structured outputs config, mirroring `create_engine_config`, which this
|
||||
# GPU-less entrypoint bypasses. Model-specific defaults are then applied
|
||||
# by `verify_and_update_config` during VllmConfig construction.
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
logging_config=engine_args.create_logging_config(),
|
||||
structured_outputs_config=engine_args.create_structured_outputs_config(),
|
||||
)
|
||||
shutdown_task = await build_and_serve_renderer(
|
||||
vllm_config, listen_address, sock, args
|
||||
)
|
||||
try:
|
||||
await shutdown_task
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvloop
|
||||
|
||||
from vllm.entrypoints.serve.utils.api_utils import cli_env_setup
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from ..cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
|
||||
cli_env_setup()
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Starts a GPU-less rendering server "
|
||||
"for preprocessing and postprocessing only"
|
||||
)
|
||||
parser = make_arg_parser(parser)
|
||||
args = parser.parse_args()
|
||||
configure_logging_from_args(args)
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
uvloop.run(run_launch_fastapi(args))
|
||||
@@ -1,672 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from argparse import Namespace
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
from io import StringIO
|
||||
from typing import Any, NoReturn, TypeAlias
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import aiohttp
|
||||
import pybase64 as base64
|
||||
import pydantic
|
||||
import torch
|
||||
from prometheus_client import start_http_server
|
||||
from pydantic import TypeAdapter, field_validator
|
||||
from pydantic_core.core_schema import ValidationInfo
|
||||
from starlette.datastructures import State
|
||||
from starlette.responses import JSONResponse
|
||||
from tqdm import tqdm
|
||||
from urllib3.util import parse_url
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import config
|
||||
from vllm.connections import HTTPResponseSizeExceededError, global_http_connection
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorResponse,
|
||||
OpenAIBaseModel,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import configure_logging_from_args, init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
from .api_server.entry import init_app_state
|
||||
from .cli_args import BaseFrontendArgs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_audio_download_limit_bytes() -> int:
|
||||
return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
|
||||
|
||||
|
||||
def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
|
||||
raise VLLMValidationError(
|
||||
"Maximum file size exceeded",
|
||||
parameter="audio_filesize_mb",
|
||||
value=size_bytes / MiB_bytes,
|
||||
)
|
||||
|
||||
|
||||
BatchRequestInputBody: TypeAlias = ChatCompletionRequest
|
||||
|
||||
|
||||
class BatchRequestInput(OpenAIBaseModel):
|
||||
"""The per-line object of the batch input file.
|
||||
|
||||
NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
|
||||
"""
|
||||
|
||||
# A developer-provided per-request id that will be used to match outputs to
|
||||
# inputs. Must be unique for each request in a batch.
|
||||
custom_id: str
|
||||
|
||||
# The HTTP method to be used for the request. Currently only POST is
|
||||
# supported.
|
||||
method: str
|
||||
|
||||
# The OpenAI API relative URL to be used for the request. Currently
|
||||
# /v1/chat/completions is supported.
|
||||
url: str
|
||||
|
||||
# The parameters of the request.
|
||||
body: BatchRequestInputBody
|
||||
|
||||
@field_validator("body", mode="plain")
|
||||
@classmethod
|
||||
def check_type_for_url(cls, value: Any, info: ValidationInfo):
|
||||
# Use url to disambiguate models
|
||||
url: str = info.data["url"]
|
||||
if url == "/v1/chat/completions":
|
||||
return ChatCompletionRequest.model_validate(value)
|
||||
return TypeAdapter(BatchRequestInputBody).validate_python(value)
|
||||
|
||||
|
||||
AllResponse: TypeAlias = (
|
||||
ChatCompletionResponse
|
||||
)
|
||||
|
||||
|
||||
class BatchResponseData(OpenAIBaseModel):
|
||||
# HTTP status code of the response.
|
||||
status_code: int = 200
|
||||
|
||||
# An unique identifier for the API request.
|
||||
request_id: str
|
||||
|
||||
# The body of the response.
|
||||
body: AllResponse | None = None
|
||||
|
||||
|
||||
class BatchRequestOutput(OpenAIBaseModel):
|
||||
"""The per-line object of the batch output and error files"""
|
||||
|
||||
id: str
|
||||
|
||||
# A developer-provided per-request id that will be used to match outputs to
|
||||
# inputs.
|
||||
custom_id: str
|
||||
|
||||
response: BatchResponseData | None
|
||||
|
||||
# For requests that failed with a non-HTTP error, this will contain more
|
||||
# information on the cause of the failure.
|
||||
error: Any | None
|
||||
|
||||
|
||||
@config
|
||||
class BatchFrontendArgs(BaseFrontendArgs):
|
||||
"""Arguments for the batch runner frontend."""
|
||||
|
||||
input_file: str | None = None
|
||||
"""The path or url to a single input file. Currently supports local file
|
||||
paths, or the http protocol (http or https). If a URL is specified,
|
||||
the file should be available via HTTP GET."""
|
||||
output_file: str | None = None
|
||||
"""The path or url to a single output file. Currently supports
|
||||
local file paths, or web (http or https) urls. If a URL is specified,
|
||||
the file should be available via HTTP PUT."""
|
||||
output_tmp_dir: str | None = None
|
||||
"""The directory to store the output file before uploading it
|
||||
to the output URL."""
|
||||
enable_metrics: bool = False
|
||||
"""Enable Prometheus metrics"""
|
||||
host: str | None = None
|
||||
"""Host name for the Prometheus metrics server
|
||||
(only needed if enable-metrics is set)."""
|
||||
port: int = 8000
|
||||
"""Port number for the Prometheus metrics server
|
||||
(only needed if enable-metrics is set)."""
|
||||
url: str = "0.0.0.0"
|
||||
"""[DEPRECATED] Host name for the Prometheus metrics server
|
||||
(only needed if enable-metrics is set). Use --host instead."""
|
||||
|
||||
@classmethod
|
||||
def _customize_cli_kwargs(
|
||||
cls,
|
||||
frontend_kwargs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
|
||||
|
||||
frontend_kwargs["input_file"]["flags"] = ["-i"]
|
||||
frontend_kwargs["input_file"]["required"] = True
|
||||
frontend_kwargs["output_file"]["flags"] = ["-o"]
|
||||
frontend_kwargs["output_file"]["required"] = True
|
||||
|
||||
frontend_kwargs["enable_metrics"]["action"] = "store_true"
|
||||
|
||||
frontend_kwargs["url"]["deprecated"] = True
|
||||
return frontend_kwargs
|
||||
|
||||
|
||||
def make_arg_parser(parser: FlexibleArgumentParser):
|
||||
parser = BatchFrontendArgs.add_cli_args(parser)
|
||||
parser = AsyncEngineArgs.add_cli_args(parser)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible batch runner.")
|
||||
args = make_arg_parser(parser).parse_args()
|
||||
configure_logging_from_args(args)
|
||||
|
||||
# Backward compatibility: If --url is set, use it for host
|
||||
url_explicit = any(arg == "--url" or arg.startswith("--url=") for arg in sys.argv)
|
||||
host_explicit = any(
|
||||
arg == "--host" or arg.startswith("--host=") for arg in sys.argv
|
||||
)
|
||||
if url_explicit and hasattr(args, "url") and not host_explicit:
|
||||
args.host = args.url
|
||||
logger.warning_once(
|
||||
"Using --url for metrics is deprecated. Please use --host instead."
|
||||
)
|
||||
|
||||
return args
|
||||
|
||||
|
||||
# explicitly use pure text format, with a newline at the end
|
||||
# this makes it impossible to see the animation in the progress bar
|
||||
# but will avoid messing up with ray or multiprocessing, which wraps
|
||||
# each line of output with some prefix.
|
||||
_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501
|
||||
|
||||
|
||||
class BatchProgressTracker:
|
||||
def __init__(self):
|
||||
self._total = 0
|
||||
self._pbar: tqdm | None = None
|
||||
|
||||
def submitted(self):
|
||||
self._total += 1
|
||||
|
||||
def completed(self):
|
||||
if self._pbar:
|
||||
self._pbar.update()
|
||||
|
||||
def pbar(self) -> tqdm:
|
||||
enable_tqdm = (
|
||||
not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
|
||||
)
|
||||
self._pbar = tqdm(
|
||||
total=self._total,
|
||||
unit="req",
|
||||
desc="Running batch",
|
||||
mininterval=5,
|
||||
disable=not enable_tqdm,
|
||||
bar_format=_BAR_FORMAT,
|
||||
)
|
||||
return self._pbar
|
||||
|
||||
|
||||
async def read_file(path_or_url: str) -> str:
|
||||
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
|
||||
async with aiohttp.ClientSession() as session, session.get(path_or_url) as resp:
|
||||
resp.raise_for_status()
|
||||
return await resp.text()
|
||||
else:
|
||||
with open(path_or_url, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
async def write_local_file(
|
||||
output_path: str, batch_outputs: list[BatchRequestOutput]
|
||||
) -> None:
|
||||
"""Write the responses to a local file.
|
||||
output_path: The path to write the responses to.
|
||||
batch_outputs: The list of batch outputs to write.
|
||||
"""
|
||||
# We should make this async, but as long as run_batch runs as a
|
||||
# standalone program, blocking the event loop won't affect performance.
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
for o in batch_outputs:
|
||||
print(o.model_dump_json(), file=f)
|
||||
|
||||
|
||||
async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> None:
|
||||
"""Upload a local file to a URL.
|
||||
output_url: The URL to upload the file to.
|
||||
data_or_file: Either the data to upload or the path to the file to upload.
|
||||
from_file: If True, data_or_file is the path to the file to upload.
|
||||
"""
|
||||
# Timeout is a common issue when uploading large files.
|
||||
# We retry max_retries times before giving up.
|
||||
max_retries = 5
|
||||
# Number of seconds to wait before retrying.
|
||||
delay = 5
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
# We increase the timeout to 1000 seconds to allow
|
||||
# for large files (default is 300).
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=1000)
|
||||
) as session:
|
||||
if from_file:
|
||||
with open(data_or_file, "rb") as file:
|
||||
async with session.put(output_url, data=file) as response:
|
||||
if not 200 <= response.status < 300:
|
||||
error_text = await response.text()
|
||||
raise Exception(
|
||||
f"Failed to upload file.\n"
|
||||
f"Status: {response.status}\n"
|
||||
f"Response: {error_text}"
|
||||
)
|
||||
else:
|
||||
async with session.put(output_url, data=data_or_file) as response:
|
||||
if not 200 <= response.status < 300:
|
||||
error_text = await response.text()
|
||||
raise Exception(
|
||||
f"Failed to upload data.\n"
|
||||
f"Status: {response.status}\n"
|
||||
f"Response: {error_text}"
|
||||
)
|
||||
# Upload succeeded; do not retry.
|
||||
return
|
||||
except Exception as e:
|
||||
if attempt < max_retries:
|
||||
logger.error(
|
||||
"Failed to upload data (attempt %d). Error message: %s.\nRetrying in %d seconds...", # noqa: E501
|
||||
attempt,
|
||||
e,
|
||||
delay,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
else:
|
||||
raise Exception(
|
||||
f"Failed to upload data (attempt {attempt}). Error message: {str(e)}." # noqa: E501
|
||||
) from e
|
||||
|
||||
|
||||
async def write_file(
|
||||
path_or_url: str, batch_outputs: list[BatchRequestOutput], output_tmp_dir: str
|
||||
) -> None:
|
||||
"""Write batch_outputs to a file or upload to a URL.
|
||||
path_or_url: The path or URL to write batch_outputs to.
|
||||
batch_outputs: The list of batch outputs to write.
|
||||
output_tmp_dir: The directory to store the output file before uploading it
|
||||
to the output URL.
|
||||
"""
|
||||
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
|
||||
if output_tmp_dir is None:
|
||||
logger.info("Writing outputs to memory buffer")
|
||||
output_buffer = StringIO()
|
||||
for o in batch_outputs:
|
||||
print(o.model_dump_json(), file=output_buffer)
|
||||
output_buffer.seek(0)
|
||||
logger.info("Uploading outputs to %s", path_or_url)
|
||||
await upload_data(
|
||||
path_or_url,
|
||||
output_buffer.read().strip().encode("utf-8"),
|
||||
from_file=False,
|
||||
)
|
||||
else:
|
||||
# Write responses to a temporary file and then upload it to the URL.
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
encoding="utf-8",
|
||||
dir=output_tmp_dir,
|
||||
prefix="tmp_batch_output_",
|
||||
suffix=".jsonl",
|
||||
) as f:
|
||||
logger.info("Writing outputs to temporary local file %s", f.name)
|
||||
await write_local_file(f.name, batch_outputs)
|
||||
logger.info("Uploading outputs to %s", path_or_url)
|
||||
await upload_data(path_or_url, f.name, from_file=True)
|
||||
else:
|
||||
logger.info("Writing outputs to local file %s", path_or_url)
|
||||
await write_local_file(path_or_url, batch_outputs)
|
||||
|
||||
|
||||
async def download_bytes_from_url(
|
||||
url: str,
|
||||
allowed_media_domains: list[str] | None = None,
|
||||
) -> bytes:
|
||||
"""Download data from a URL or decode from a data URL.
|
||||
|
||||
Args:
|
||||
url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
|
||||
allowed_media_domains: If set, only HTTP/HTTPS URLs whose hostname
|
||||
is in this list are permitted. data: URLs are not subject to
|
||||
this restriction.
|
||||
|
||||
Returns:
|
||||
Data as bytes
|
||||
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
|
||||
# Handle data URLs (base64 encoded) - not subject to domain restrictions
|
||||
if parsed.scheme == "data":
|
||||
# Format: data:...;base64,<base64_data>
|
||||
if "," in url:
|
||||
header, data = url.split(",", 1)
|
||||
if "base64" in header:
|
||||
decoded_size_bytes = len(data.rstrip("=")) * 3 // 4
|
||||
if decoded_size_bytes > _get_audio_download_limit_bytes():
|
||||
_raise_audio_filesize_limit(decoded_size_bytes)
|
||||
return base64.b64decode(data)
|
||||
else:
|
||||
raise VLLMValidationError(
|
||||
f"Unsupported data URL encoding: {header}",
|
||||
parameter="url",
|
||||
)
|
||||
else:
|
||||
raise VLLMValidationError(
|
||||
f"Invalid data URL format: {url}",
|
||||
parameter="url",
|
||||
)
|
||||
|
||||
# Handle HTTP/HTTPS URLs
|
||||
elif parsed.scheme in ("http", "https"):
|
||||
if allowed_media_domains is not None:
|
||||
url_spec = parse_url(url)
|
||||
if url_spec.hostname not in allowed_media_domains:
|
||||
raise VLLMValidationError(
|
||||
f"The URL must be from one of the allowed domains: "
|
||||
f"{allowed_media_domains}. Input URL domain: "
|
||||
f"{url_spec.hostname}",
|
||||
parameter="url",
|
||||
value=url_spec.hostname,
|
||||
)
|
||||
# Use the normalized URL to prevent parsing discrepancies
|
||||
# between urllib3 and aiohttp (e.g. backslash-@ attacks).
|
||||
url = url_spec.url
|
||||
|
||||
try:
|
||||
return await global_http_connection.async_get_bytes(
|
||||
url,
|
||||
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
||||
max_bytes=_get_audio_download_limit_bytes(),
|
||||
)
|
||||
except HTTPResponseSizeExceededError as exc:
|
||||
_raise_audio_filesize_limit(exc.received_bytes)
|
||||
|
||||
else:
|
||||
raise VLLMValidationError(
|
||||
f"Unsupported URL scheme: {parsed.scheme}. "
|
||||
"Supported schemes: http, https, data",
|
||||
parameter="url",
|
||||
value=parsed.scheme,
|
||||
)
|
||||
|
||||
|
||||
def make_error_request_output(
|
||||
request: BatchRequestInput, error_msg: str
|
||||
) -> BatchRequestOutput:
|
||||
batch_output = BatchRequestOutput(
|
||||
id=f"vllm-{random_uuid()}",
|
||||
custom_id=request.custom_id,
|
||||
response=BatchResponseData(
|
||||
status_code=HTTPStatus.BAD_REQUEST,
|
||||
request_id=f"vllm-batch-{random_uuid()}",
|
||||
),
|
||||
error=error_msg,
|
||||
)
|
||||
return batch_output
|
||||
|
||||
|
||||
async def make_async_error_request_output(
|
||||
request: BatchRequestInput, error_msg: str
|
||||
) -> BatchRequestOutput:
|
||||
return make_error_request_output(request, error_msg)
|
||||
|
||||
|
||||
async def run_request(
|
||||
serving_engine_func: Callable,
|
||||
request: BatchRequestInput,
|
||||
tracker: BatchProgressTracker,
|
||||
) -> BatchRequestOutput:
|
||||
try:
|
||||
response = await serving_engine_func(request.body)
|
||||
except Exception as e:
|
||||
response = create_error_response(e)
|
||||
|
||||
if isinstance(response, JSONResponse):
|
||||
with contextlib.suppress(pydantic.ValidationError):
|
||||
response = TypeAdapter(AllResponse | ErrorResponse).validate_python(
|
||||
json.loads(response.body)
|
||||
)
|
||||
|
||||
if isinstance(response, AllResponse):
|
||||
batch_output = BatchRequestOutput(
|
||||
id=f"vllm-{random_uuid()}",
|
||||
custom_id=request.custom_id,
|
||||
response=BatchResponseData(
|
||||
body=response, request_id=f"vllm-batch-{random_uuid()}"
|
||||
),
|
||||
error=None,
|
||||
)
|
||||
elif isinstance(response, ErrorResponse):
|
||||
batch_output = BatchRequestOutput(
|
||||
id=f"vllm-{random_uuid()}",
|
||||
custom_id=request.custom_id,
|
||||
response=BatchResponseData(
|
||||
status_code=response.error.code,
|
||||
request_id=f"vllm-batch-{random_uuid()}",
|
||||
),
|
||||
error=response,
|
||||
)
|
||||
else:
|
||||
batch_output = make_error_request_output(
|
||||
request, error_msg="Request must not be sent in stream mode"
|
||||
)
|
||||
|
||||
tracker.completed()
|
||||
return batch_output
|
||||
|
||||
|
||||
WrapperFn: TypeAlias = Callable[[Callable], Callable]
|
||||
|
||||
|
||||
def handle_endpoint_request(
|
||||
request: BatchRequestInput,
|
||||
tracker: BatchProgressTracker,
|
||||
url_matcher: Callable[[str], bool],
|
||||
handler_getter: Callable[[], Callable | None],
|
||||
wrapper_fn: WrapperFn | None = None,
|
||||
) -> Awaitable[BatchRequestOutput] | None:
|
||||
"""Generic handler for endpoint requests.
|
||||
|
||||
Args:
|
||||
request: The batch request input
|
||||
tracker: Progress tracker for the batch
|
||||
url_matcher: Function that takes a URL and returns True if it matches
|
||||
handler_getter: Function that returns the handler function or None
|
||||
wrapper_fn: Optional function to wrap the handler (e.g., for transcriptions)
|
||||
|
||||
Returns:
|
||||
Awaitable[BatchRequestOutput] if the request was handled,
|
||||
None if URL didn't match
|
||||
|
||||
"""
|
||||
if not url_matcher(request.url):
|
||||
return None
|
||||
|
||||
handler_fn = handler_getter()
|
||||
if handler_fn is None:
|
||||
error_msg = f"Model does not support endpoint: {request.url}"
|
||||
return make_async_error_request_output(request, error_msg=error_msg)
|
||||
|
||||
# Apply wrapper if provided (e.g., for transcriptions/translations)
|
||||
if wrapper_fn is not None:
|
||||
handler_fn = wrapper_fn(handler_fn)
|
||||
|
||||
tracker.submitted()
|
||||
return run_request(handler_fn, request, tracker)
|
||||
|
||||
|
||||
|
||||
async def build_endpoint_registry(
|
||||
engine_client: EngineClient,
|
||||
args: Namespace,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Build the endpoint registry with all serving objects and handler configurations.
|
||||
|
||||
Args:
|
||||
engine_client: The engine client
|
||||
args: Command line arguments
|
||||
|
||||
Returns:
|
||||
Dictionary mapping endpoint keys to their configurations
|
||||
|
||||
"""
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
|
||||
# Create a state object to hold serving objects
|
||||
state = State()
|
||||
|
||||
# Initialize all serving objects using init_app_state
|
||||
# This provides full functionality including chat template processing,
|
||||
# LoRA support, tool servers, etc.
|
||||
await init_app_state(engine_client, state, args, supported_tasks)
|
||||
|
||||
# Get serving objects from state (defaulting to None if not set)
|
||||
openai_serving_chat = getattr(state, "openai_serving_chat", None)
|
||||
|
||||
# Registry of endpoint configurations
|
||||
endpoint_registry: dict[str, dict[str, Any]] = {
|
||||
"completions": {
|
||||
"url_matcher": lambda url: url == "/v1/chat/completions",
|
||||
"handler_getter": lambda: (
|
||||
openai_serving_chat.create_chat_completion
|
||||
if openai_serving_chat is not None
|
||||
else None
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
}
|
||||
|
||||
return endpoint_registry
|
||||
|
||||
|
||||
def validate_run_batch_args(args):
|
||||
valid_reasoning_parsers = ReasoningParserManager.list_registered()
|
||||
if (
|
||||
reasoning_parser := args.structured_outputs_config.reasoning_parser
|
||||
) and reasoning_parser not in valid_reasoning_parsers:
|
||||
raise KeyError(
|
||||
f"invalid reasoning parser: {reasoning_parser} "
|
||||
f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
|
||||
)
|
||||
|
||||
|
||||
async def run_batch(
|
||||
engine_client: EngineClient,
|
||||
args: Namespace,
|
||||
) -> None:
|
||||
endpoint_registry = await build_endpoint_registry(
|
||||
engine_client=engine_client,
|
||||
args=args,
|
||||
)
|
||||
|
||||
tracker = BatchProgressTracker()
|
||||
logger.info("Reading batch from %s...", args.input_file)
|
||||
|
||||
# Submit all requests in the file to the engine "concurrently".
|
||||
response_futures: list[Awaitable[BatchRequestOutput]] = []
|
||||
for request_json in (await read_file(args.input_file)).strip().split("\n"):
|
||||
# Skip empty lines.
|
||||
request_json = request_json.strip()
|
||||
if not request_json:
|
||||
continue
|
||||
|
||||
request = BatchRequestInput.model_validate_json(request_json)
|
||||
|
||||
# Use the last segment of the URL as the endpoint key.
|
||||
# More advanced URL matching is done in url_matcher of endpoint_registry.
|
||||
endpoint_key = request.url.split("/")[-1]
|
||||
|
||||
result = None
|
||||
if endpoint_key in endpoint_registry:
|
||||
endpoint_config = endpoint_registry[endpoint_key]
|
||||
result = handle_endpoint_request(
|
||||
request,
|
||||
tracker,
|
||||
url_matcher=endpoint_config["url_matcher"],
|
||||
handler_getter=endpoint_config["handler_getter"],
|
||||
wrapper_fn=endpoint_config["wrapper_fn"],
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
response_futures.append(result)
|
||||
else:
|
||||
response_futures.append(
|
||||
make_async_error_request_output(
|
||||
request,
|
||||
error_msg=f"URL {request.url} was used. "
|
||||
"Supported endpoints: /v1/chat/completions.",
|
||||
)
|
||||
)
|
||||
|
||||
with tracker.pbar():
|
||||
responses = await asyncio.gather(*response_futures)
|
||||
|
||||
await write_file(args.output_file, responses, args.output_tmp_dir)
|
||||
|
||||
|
||||
async def main(args: Namespace):
|
||||
from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
|
||||
validate_run_batch_args(args)
|
||||
|
||||
async with build_async_engine_client(
|
||||
args,
|
||||
usage_context=UsageContext.OPENAI_BATCH_RUNNER,
|
||||
) as engine_client:
|
||||
await run_batch(engine_client, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
|
||||
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
|
||||
logger.info("args: %s", args)
|
||||
|
||||
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
|
||||
# to publish metrics at the /metrics endpoint.
|
||||
if args.enable_metrics:
|
||||
logger.info("Prometheus metrics enabled")
|
||||
start_http_server(port=args.port, addr=args.host)
|
||||
else:
|
||||
logger.info("Prometheus metrics disabled")
|
||||
|
||||
asyncio.run(main(args))
|
||||
@@ -30,7 +30,6 @@ from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.common.offline import _O, _R, OfflineInferenceMixin
|
||||
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
|
||||
from vllm.entrypoints.rl.offline import RLOfflineMixin
|
||||
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.logger import configure_logging_if_needed, init_logger
|
||||
@@ -59,7 +58,7 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LLM(BeamSearchOfflineMixin, RLOfflineMixin, OfflineInferenceMixin):
|
||||
class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
"""An LLM for generating texts from given prompts and sampling parameters.
|
||||
|
||||
This class includes a tokenizer, a language model (possibly distributed
|
||||
|
||||
@@ -18,8 +18,6 @@ from vllm.entrypoints.launchers.launcher import (
|
||||
setup_server,
|
||||
validate_api_server_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.render.app_state import init_render_app_state
|
||||
from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer
|
||||
|
||||
warnings.warn(
|
||||
"`vllm.entrypoints.openai.api_server` is deprecated and will likely be"
|
||||
@@ -35,13 +33,11 @@ __all__ = [
|
||||
"build_async_engine_client_from_engine_args",
|
||||
"build_app",
|
||||
"init_app_state",
|
||||
"init_render_app_state",
|
||||
"create_server_socket",
|
||||
"create_server_unix_socket",
|
||||
"validate_api_server_args",
|
||||
"setup_server",
|
||||
"build_and_serve",
|
||||
"build_and_serve_renderer",
|
||||
"run_server",
|
||||
"run_server_worker",
|
||||
"register_api_routers",
|
||||
|
||||
@@ -107,7 +107,6 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
|
||||
class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
|
||||
# so that added fields like ``citations`` survive JSON serialization
|
||||
# instead of being stripped down to the base schema. Plain
|
||||
# ``ChatMessage`` instances serialize identically to before.
|
||||
@@ -162,7 +161,6 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
|
||||
# so streaming ``citations`` survive JSON serialization. Plain
|
||||
# ``DeltaMessage`` instances serialize identically to before.
|
||||
delta: SerializeAsAny[DeltaMessage]
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Translate Decisions questions and answers to and from label reads."""
|
||||
|
||||
import json
|
||||
|
||||
from .protocol import (
|
||||
ChoiceAnswer,
|
||||
ChoiceQuestion,
|
||||
DecisionAnswer,
|
||||
DecisionInputMessage,
|
||||
DecisionQuestion,
|
||||
PredicateAnswer,
|
||||
PredicateQuestion,
|
||||
ScoreAnswer,
|
||||
)
|
||||
from .question_types import (
|
||||
LABELS,
|
||||
Option,
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
argmax,
|
||||
)
|
||||
from .question_types import (
|
||||
ChoiceQuestion as LabelQuestion,
|
||||
)
|
||||
|
||||
|
||||
def make_read_question(
|
||||
index: int, question: DecisionQuestion, max_options: int = len(LABELS)
|
||||
) -> Question:
|
||||
if isinstance(question, PredicateQuestion):
|
||||
options = [Option("false"), Option("true")]
|
||||
elif isinstance(question, ChoiceQuestion):
|
||||
options = [
|
||||
Option(json.dumps(c.value, ensure_ascii=False), c.description)
|
||||
for c in question.choices
|
||||
]
|
||||
else:
|
||||
options = [Option(level.label, level.description) for level in question.levels]
|
||||
limit = min(max_options, len(LABELS))
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"This model supports at most {limit} choices per question"
|
||||
)
|
||||
return Question(
|
||||
id=str(index),
|
||||
type=LabelQuestion(),
|
||||
instructions=question.instructions,
|
||||
options=tuple(options),
|
||||
labels=LABELS[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
def make_answer(
|
||||
question: DecisionQuestion, probs: list[float], label_mass: float
|
||||
) -> DecisionAnswer:
|
||||
if isinstance(question, PredicateQuestion):
|
||||
return PredicateAnswer(name=question.name, probability=probs[1])
|
||||
top = argmax(probs)
|
||||
confidence = probs[top] * label_mass
|
||||
if isinstance(question, ChoiceQuestion):
|
||||
return ChoiceAnswer(
|
||||
name=question.name,
|
||||
choice=question.choices[top].value,
|
||||
probabilities=[
|
||||
{"value": choice.value, "probability": probability}
|
||||
for choice, probability in zip(question.choices, probs)
|
||||
],
|
||||
confidence=confidence,
|
||||
)
|
||||
return ScoreAnswer(
|
||||
name=question.name,
|
||||
score=sum(i * probability for i, probability in enumerate(probs)),
|
||||
probabilities=[
|
||||
{"value": i, "label": level.label, "probability": probability}
|
||||
for i, (level, probability) in enumerate(zip(question.levels, probs))
|
||||
],
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
|
||||
def input_text(input: str | list[DecisionInputMessage]) -> str:
|
||||
if isinstance(input, str):
|
||||
return input
|
||||
return "\n\n".join(
|
||||
message.content
|
||||
if isinstance(message.content, str)
|
||||
else "\n".join(part.text for part in message.content)
|
||||
for message in input
|
||||
)
|
||||
@@ -1,56 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import DecisionRequest, DecisionResponse
|
||||
from .serving import OpenAIServingDecisions
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/decisions",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=DecisionResponse,
|
||||
responses={
|
||||
status.value: {"model": ErrorResponse}
|
||||
for status in (
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
HTTPStatus.NOT_FOUND,
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
HTTPStatus.NOT_IMPLEMENTED,
|
||||
)
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_decision(request: DecisionRequest, raw_request: Request):
|
||||
handler: OpenAIServingDecisions | None = getattr(
|
||||
raw_request.app.state, "openai_serving_decisions", None
|
||||
)
|
||||
if handler is None:
|
||||
error = create_error_response(
|
||||
NotImplementedError("The model does not support the Decisions API")
|
||||
)
|
||||
return JSONResponse(content=error.model_dump(), status_code=error.error.code)
|
||||
result = await handler.create_decisions(request, raw_request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def register_decisions_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -1,181 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""OpenAI Decisions API request and response models."""
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
|
||||
ShortText = Annotated[StrictStr, Field(max_length=1048576)]
|
||||
InputText = Annotated[StrictStr, Field(max_length=10485760)]
|
||||
ChoiceValue = StrictStr | StrictBool
|
||||
|
||||
|
||||
class DecisionModel(OpenAIBaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DecisionInputText(DecisionModel):
|
||||
type: Literal["input_text"]
|
||||
text: InputText
|
||||
|
||||
|
||||
class DecisionInputMessage(DecisionModel):
|
||||
role: Literal["user"]
|
||||
content: InputText | Annotated[list[DecisionInputText], Field(max_length=16384)]
|
||||
type: Literal["message"] = "message"
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def reject_images(cls, data: Any) -> Any:
|
||||
if (
|
||||
isinstance(data, dict)
|
||||
and isinstance(data.get("content"), list)
|
||||
and any(
|
||||
isinstance(part, dict) and part.get("type") == "input_image"
|
||||
for part in data["content"]
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Decisions supports text input only; images are not supported"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
class QuestionBase(DecisionModel):
|
||||
instructions: ShortText
|
||||
name: ShortText | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_name(self) -> "QuestionBase":
|
||||
if "name" in self.model_fields_set and self.name is None:
|
||||
raise ValueError("name must be a string when provided")
|
||||
return self
|
||||
|
||||
|
||||
class PredicateQuestion(QuestionBase):
|
||||
type: Literal["predicate"]
|
||||
|
||||
|
||||
class ChoiceOption(DecisionModel):
|
||||
value: ChoiceValue
|
||||
description: ShortText = ""
|
||||
|
||||
|
||||
class ChoiceQuestion(QuestionBase):
|
||||
type: Literal["choice"]
|
||||
choices: list[ChoiceOption] = Field(min_length=2, max_length=255)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_choices(self) -> "ChoiceQuestion":
|
||||
values = [(type(c.value), c.value) for c in self.choices]
|
||||
if len(set(values)) != len(values):
|
||||
raise ValueError("choice values must be distinct")
|
||||
return self
|
||||
|
||||
|
||||
class ScoreLevel(DecisionModel):
|
||||
label: ShortText
|
||||
description: ShortText = ""
|
||||
|
||||
|
||||
class ScoreQuestion(QuestionBase):
|
||||
type: Literal["score"]
|
||||
levels: list[ScoreLevel] = Field(min_length=2, max_length=10)
|
||||
|
||||
|
||||
DecisionQuestion = Annotated[
|
||||
PredicateQuestion | ChoiceQuestion | ScoreQuestion, Field(discriminator="type")
|
||||
]
|
||||
|
||||
|
||||
class DecisionRequest(DecisionModel):
|
||||
model: ShortText
|
||||
input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)]
|
||||
questions: list[DecisionQuestion] = Field(min_length=1, max_length=200)
|
||||
safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=1024,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
|
||||
class PredicateAnswer(DecisionModel):
|
||||
type: Literal["predicate"] = "predicate"
|
||||
name: str | None
|
||||
probability: float
|
||||
|
||||
|
||||
class ChoiceProbability(DecisionModel):
|
||||
value: ChoiceValue
|
||||
probability: float
|
||||
|
||||
|
||||
class ChoiceAnswer(DecisionModel):
|
||||
type: Literal["choice"] = "choice"
|
||||
name: str | None
|
||||
choice: ChoiceValue
|
||||
probabilities: list[ChoiceProbability]
|
||||
confidence: float
|
||||
|
||||
|
||||
class ScoreProbability(DecisionModel):
|
||||
value: int
|
||||
label: str
|
||||
probability: float
|
||||
|
||||
|
||||
class ScoreAnswer(DecisionModel):
|
||||
type: Literal["score"] = "score"
|
||||
name: str | None
|
||||
score: float
|
||||
probabilities: list[ScoreProbability]
|
||||
confidence: float
|
||||
|
||||
|
||||
DecisionAnswer = Annotated[
|
||||
PredicateAnswer | ChoiceAnswer | ScoreAnswer,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class InputTokensDetails(DecisionModel):
|
||||
cached_tokens: int = 0
|
||||
cache_write_tokens: int = 0
|
||||
|
||||
|
||||
class OutputTokensDetails(DecisionModel):
|
||||
reasoning_tokens: int = 0
|
||||
|
||||
|
||||
class OpenAIDecisionUsage(DecisionModel):
|
||||
input_tokens: int
|
||||
input_tokens_details: InputTokensDetails
|
||||
output_tokens: int
|
||||
output_tokens_details: OutputTokensDetails
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class DecisionResponse(DecisionModel):
|
||||
model: str
|
||||
answers: list[DecisionAnswer]
|
||||
usage: OpenAIDecisionUsage
|
||||
@@ -1,240 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Question types for structured decisions.
|
||||
|
||||
A question type turns a request's criteria into options, writes the question
|
||||
for the model with a label per option, and builds the answer from the label
|
||||
probabilities.
|
||||
"""
|
||||
|
||||
import math
|
||||
import string
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar
|
||||
|
||||
|
||||
class StructuredDecisionError(ValueError):
|
||||
"""An invalid request. Returned as a 400."""
|
||||
|
||||
|
||||
#: A choice labels its options A to Z in order. A single capital letter is one
|
||||
#: token at the start of a reply for the tokenizers tested. Every read checks it.
|
||||
LABELS = tuple(string.ascii_uppercase)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Option:
|
||||
name: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Question:
|
||||
id: str
|
||||
type: "QuestionType"
|
||||
instructions: str
|
||||
options: tuple[Option, ...]
|
||||
labels: tuple[str, ...]
|
||||
|
||||
|
||||
class QuestionType(ABC):
|
||||
name: ClassVar[str]
|
||||
#: The labels of a question's options, in option order.
|
||||
label_set: ClassVar[tuple[str, ...]] = LABELS
|
||||
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
|
||||
|
||||
@abstractmethod
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
|
||||
|
||||
@abstractmethod
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
"""The answer for ``question``. ``probs[i]`` is the probability of
|
||||
``question.labels[i]`` among the labels, and the list sums to 1.
|
||||
``label_mass`` is the labels' total probability over the vocabulary."""
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str:
|
||||
return (
|
||||
f"{label}: {option.name} - {option.description}"
|
||||
if option.description
|
||||
else f"{label}: {option.name}"
|
||||
)
|
||||
|
||||
def prompt(self, question: Question) -> str:
|
||||
"""The question as the model reads it, after the state."""
|
||||
lines = [f"Question: {question.instructions}"] if question.instructions else []
|
||||
for label, o in zip(question.labels, question.options):
|
||||
lines.append(self.reply_line(label, o))
|
||||
lines.append(self.reply_instruction)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
QUESTION_TYPES: dict[str, QuestionType] = {}
|
||||
|
||||
|
||||
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
|
||||
if cls.name in QUESTION_TYPES:
|
||||
raise ValueError(f"question type {cls.name!r} is already registered")
|
||||
QUESTION_TYPES[cls.name] = cls()
|
||||
return cls
|
||||
|
||||
|
||||
def get_question_type(name: str) -> QuestionType:
|
||||
try:
|
||||
return QUESTION_TYPES[name]
|
||||
except KeyError:
|
||||
raise StructuredDecisionError(
|
||||
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
|
||||
) from None
|
||||
|
||||
|
||||
def build_question(
|
||||
qid: str,
|
||||
type_name: str,
|
||||
instructions: Any,
|
||||
criteria: Any,
|
||||
max_options: int,
|
||||
labeled: bool = True,
|
||||
) -> Question:
|
||||
if not qid:
|
||||
raise StructuredDecisionError("question ids must be non-empty")
|
||||
qtype = get_question_type(type_name)
|
||||
options = qtype.parse_options(qid, criteria)
|
||||
if not options:
|
||||
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
|
||||
names = [o.name for o in options]
|
||||
if len(set(names)) != len(names):
|
||||
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
|
||||
limit = min(max_options, len(qtype.label_set)) if labeled else max_options
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: at most {limit} options for this model"
|
||||
)
|
||||
if not isinstance(instructions, str):
|
||||
instructions = "" if instructions is None else str(instructions)
|
||||
return Question(
|
||||
id=qid,
|
||||
type=qtype,
|
||||
instructions=instructions,
|
||||
options=tuple(options),
|
||||
labels=qtype.label_set[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
def argmax(values: list[float]) -> int:
|
||||
return max(range(len(values)), key=values.__getitem__)
|
||||
|
||||
|
||||
def label_softmax(logprobs: list[float]) -> list[float]:
|
||||
"""Softmax of the label logprobs."""
|
||||
top = max(logprobs)
|
||||
weights = [math.exp(lp - top) for lp in logprobs]
|
||||
total = sum(weights)
|
||||
return [w / total for w in weights]
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ChoiceQuestion(QuestionType):
|
||||
"""Pick one option. ``criteria`` maps each option name to a description,
|
||||
or to null."""
|
||||
|
||||
name = "choice"
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, dict) or not criteria:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: choice criteria must map option names to "
|
||||
"a description or null"
|
||||
)
|
||||
return [
|
||||
Option(str(name), None if desc is None else str(desc))
|
||||
for name, desc in criteria.items()
|
||||
]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
"choice": question.options[top].name,
|
||||
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
|
||||
# The chosen label's probability over the whole vocabulary, so a
|
||||
# read that mostly wanted a non-label token reports low confidence.
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class NoulQuestion(QuestionType):
|
||||
"""Yes or no. ``criteria`` may describe what true and false mean."""
|
||||
|
||||
name = "noul"
|
||||
label_set = ("yes", "no")
|
||||
reply_instruction = "Answer with yes or no only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if criteria is not None and (
|
||||
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
|
||||
):
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: noul criteria must be an object with true and false"
|
||||
)
|
||||
criteria = criteria or {}
|
||||
true = criteria.get("true")
|
||||
false = criteria.get("false")
|
||||
return [
|
||||
Option("yes", None if true is None else str(true)),
|
||||
Option("no", None if false is None else str(false)),
|
||||
]
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str:
|
||||
# A side without a description still shows its label. Without the
|
||||
# label in the prompt, models answer "Yes" or "No" instead.
|
||||
return f"{label}: {option.description}" if option.description else label
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"type": self.name,
|
||||
"noul": probs[0],
|
||||
"probabilities": {"yes": probs[0], "no": probs[1]},
|
||||
# The chosen label's probability over the whole vocabulary, like a
|
||||
# choice's confidence.
|
||||
"confidence": max(probs) * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ScoreQuestion(QuestionType):
|
||||
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
|
||||
level names, from the first to the last level of the scale."""
|
||||
|
||||
name = "score"
|
||||
# 0-indexed like the answer, so a level's label is its score.
|
||||
label_set = tuple(string.digits)
|
||||
reply_instruction = "Answer with the number of one level only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, list) or len(criteria) < 2:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: score criteria must be an ordered list of levels"
|
||||
)
|
||||
return [Option(str(level)) for level in criteria]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
# The expected level, 0-indexed, so a level's index in the request's
|
||||
# criteria is its value.
|
||||
"score": sum(i * p for i, p in enumerate(probs)),
|
||||
"legend": {str(i): o.name for i, o in enumerate(question.options)},
|
||||
"probabilities": {str(i): p for i, p in enumerate(probs)},
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
from .adapters import input_text, make_answer, make_read_question
|
||||
from .protocol import (
|
||||
DecisionRequest,
|
||||
DecisionResponse,
|
||||
InputTokensDetails,
|
||||
OpenAIDecisionUsage,
|
||||
OutputTokensDetails,
|
||||
)
|
||||
from .question_types import StructuredDecisionError
|
||||
from .strategies import ReadStrategy
|
||||
|
||||
|
||||
class BaseServingDecisions(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels,
|
||||
strategy: ReadStrategy,
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=strategy.context.engine_client.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
self.strategy = strategy
|
||||
self.limits = strategy.limits()
|
||||
|
||||
|
||||
class OpenAIServingDecisions(BaseServingDecisions):
|
||||
async def create_decisions(
|
||||
self, request: DecisionRequest, raw_request: Request | None = None
|
||||
) -> DecisionResponse | ErrorResponse:
|
||||
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
|
||||
return error
|
||||
engine = self.strategy.context.engine_client
|
||||
if engine.errored:
|
||||
raise engine.dead_error
|
||||
request_id = f"decision-{self._base_request_id(raw_request)}"
|
||||
try:
|
||||
if len(request.questions) > self.limits.max_questions:
|
||||
raise StructuredDecisionError(
|
||||
f"questions: at most {self.limits.max_questions} for this model"
|
||||
)
|
||||
questions = [
|
||||
make_read_question(i, question, self.limits.max_options)
|
||||
for i, question in enumerate(request.questions)
|
||||
]
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine.check_admission(len(questions))
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
text = input_text(request.input)
|
||||
self._log_inputs(request_id, text, None, lora_request)
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
None,
|
||||
text,
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=None,
|
||||
lora_request=lora_request,
|
||||
priority=0,
|
||||
cache_salt=request.cache_salt,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
except StructuredDecisionError as exc:
|
||||
return self.create_error_response(exc)
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
input_tokens = sum(read.input_tokens for read in reads)
|
||||
output_tokens = sum(read.output_tokens for read in reads)
|
||||
return DecisionResponse(
|
||||
model=self.models.model_name(lora_request),
|
||||
answers=[
|
||||
make_answer(question, read.probs, read.label_mass)
|
||||
for question, read in zip(request.questions, reads)
|
||||
],
|
||||
usage=OpenAIDecisionUsage(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=input_tokens + output_tokens,
|
||||
input_tokens_details=InputTokensDetails(
|
||||
cached_tokens=sum(read.cached_tokens for read in reads),
|
||||
cache_write_tokens=sum(read.cache_write_tokens for read in reads),
|
||||
),
|
||||
output_tokens_details=OutputTokensDetails(),
|
||||
),
|
||||
)
|
||||
@@ -1,46 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from typing import Any
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
from .serving import OpenAIServingDecisions
|
||||
from .strategies import ReadContext, ReadStrategy, select_read_strategy
|
||||
|
||||
|
||||
def init_decisions_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
) -> ReadStrategy | None:
|
||||
state.openai_serving_decisions = None
|
||||
try:
|
||||
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
|
||||
strategy = strategy_cls(
|
||||
ReadContext(
|
||||
engine_client=engine_client,
|
||||
online_renderer=state.online_renderer,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
if strategy.task == "generate":
|
||||
state.openai_serving_decisions = OpenAIServingDecisions(
|
||||
state.openai_serving_models,
|
||||
strategy,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
return strategy
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import warnings
|
||||
|
||||
from prometheus_client import start_http_server
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
warnings.warn(
|
||||
"The `python -m vllm.entrypoints.openai.run_batch` command is deprecated "
|
||||
"and may be removed in a future release. Please use `vllm run_batch` instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=1,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from vllm.entrypoints.launchers.run_batch import main, parse_args
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
args = parse_args()
|
||||
|
||||
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
|
||||
logger.info("args: %s", args)
|
||||
|
||||
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
|
||||
# to publish metrics at the /metrics endpoint.
|
||||
if args.enable_metrics:
|
||||
logger.info("Prometheus metrics enabled")
|
||||
start_http_server(port=args.port, addr=args.host)
|
||||
else:
|
||||
logger.info("Prometheus metrics disabled")
|
||||
|
||||
asyncio.run(main(args))
|
||||
@@ -1,11 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
|
||||
def register_rl_api_routers(app: FastAPI):
|
||||
from .online.api_router import router as rl_router
|
||||
|
||||
app.include_router(rl_router)
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdateRequest,
|
||||
)
|
||||
from vllm.entrypoints.common.offline import OfflineInferenceMixin
|
||||
|
||||
|
||||
class RLOfflineMixin(OfflineInferenceMixin):
|
||||
def init_weight_transfer_engine(
|
||||
self, request: WeightTransferInitRequest | dict
|
||||
) -> None:
|
||||
"""Initialize weight transfer for RL training.
|
||||
|
||||
Args:
|
||||
request: Weight transfer initialization request with backend-specific info
|
||||
|
||||
"""
|
||||
init_info_dict = (
|
||||
request["init_info"] if isinstance(request, dict) else request.init_info
|
||||
)
|
||||
|
||||
self.llm_engine.collective_rpc(
|
||||
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
|
||||
)
|
||||
|
||||
def shutdown_weight_transfer_engine(self) -> None:
|
||||
"""Release the trainer connection without shutting down inference."""
|
||||
self.llm_engine.collective_rpc("shutdown_weight_transfer_engine")
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Start a new weight update."""
|
||||
self.llm_engine.collective_rpc("start_weight_update")
|
||||
|
||||
def start_draft_weight_update(self) -> None:
|
||||
"""Start a new weight update targeting the speculative draft model."""
|
||||
self.llm_engine.collective_rpc("start_draft_weight_update")
|
||||
|
||||
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
|
||||
"""Update the weights of the model.
|
||||
|
||||
Args:
|
||||
request: Weight update request with backend-specific update info
|
||||
|
||||
"""
|
||||
update_info_dict = (
|
||||
request["update_info"] if isinstance(request, dict) else request.update_info
|
||||
)
|
||||
|
||||
self.llm_engine.collective_rpc(
|
||||
"update_weights", kwargs={"update_info": update_info_dict}
|
||||
)
|
||||
|
||||
def finish_weight_update(self, weight_version: str | None = None) -> None:
|
||||
"""Finish the weight update and set its version if provided."""
|
||||
self.llm_engine.collective_rpc("finish_weight_update")
|
||||
if weight_version is not None:
|
||||
self.llm_engine.set_weight_version(weight_version)
|
||||
|
||||
def update_weight_version(self, new_version: str) -> None:
|
||||
"""Set the weight version without updating weights."""
|
||||
self.llm_engine.set_weight_version(new_version)
|
||||
|
||||
def get_weight_version(self) -> str:
|
||||
"""Return the latest committed weight version."""
|
||||
return self.llm_engine.get_weight_version()
|
||||
@@ -1,286 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Body, HTTPException, Query, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdatePayload,
|
||||
WeightTransferUpdateRequest,
|
||||
)
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.rl.online.metrics import weight_operation_metrics
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine import PauseMode
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/pause")
|
||||
async def pause_generation(
|
||||
raw_request: Request,
|
||||
mode: Annotated[PauseMode, Query()] = "abort",
|
||||
clear_cache: Annotated[bool, Query()] = True,
|
||||
) -> JSONResponse:
|
||||
"""Pause generation requests to allow weight updates.
|
||||
|
||||
Args:
|
||||
raw_request: The incoming FastAPI request, used to reach the engine
|
||||
client on the app state.
|
||||
mode: How to handle in-flight requests:
|
||||
- ``"abort"``: Abort all in-flight requests immediately (default).
|
||||
- ``"wait"``: Wait for in-flight requests to complete.
|
||||
- ``"keep"``: Freeze requests in queue; they resume on /resume.
|
||||
clear_cache: DEPRECATED. Whether to clear KV/prefix caches after
|
||||
draining. Ignored when mode="keep".
|
||||
|
||||
"""
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
await engine.pause_generation(
|
||||
mode=mode,
|
||||
clear_cache=clear_cache,
|
||||
)
|
||||
return JSONResponse(
|
||||
content={"status": "paused"},
|
||||
status_code=HTTPStatus.OK.value,
|
||||
)
|
||||
|
||||
except ValueError as err:
|
||||
return JSONResponse(
|
||||
content={"error": str(err)},
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
)
|
||||
except Exception as err: # pragma: no cover - defensive
|
||||
logger.exception("Failed to pause generation")
|
||||
return JSONResponse(
|
||||
content={"error": f"Failed to pause generation: {err}"},
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/resume")
|
||||
async def resume_generation(raw_request: Request) -> JSONResponse:
|
||||
"""Resume generation after a pause."""
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
await engine.resume_generation()
|
||||
return JSONResponse(
|
||||
content={"status": "resumed"},
|
||||
status_code=HTTPStatus.OK.value,
|
||||
)
|
||||
except Exception as err: # pragma: no cover - defensive
|
||||
logger.exception("Failed to resume generation")
|
||||
return JSONResponse(
|
||||
content={"error": f"Failed to resume generation: {err}"},
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/abort_requests")
|
||||
async def abort_requests(raw_request: Request) -> JSONResponse:
|
||||
"""Abort in-flight requests without pausing the scheduler.
|
||||
|
||||
Empty/missing ``request_ids`` aborts all in-flight requests.
|
||||
"""
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904
|
||||
if not isinstance(body, dict):
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Request body must be a JSON object"
|
||||
)
|
||||
|
||||
request_ids = body.get("request_ids")
|
||||
|
||||
try:
|
||||
if request_ids:
|
||||
# Body ids are external (user-supplied) request ids.
|
||||
await engine.abort(request_ids)
|
||||
else:
|
||||
# The dev RL server runs AsyncLLM; abort everything it is tracking.
|
||||
# request_states is keyed by internal ids; parent_requests holds
|
||||
# parallel-sampling parents. Abort both as internal ids.
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
assert isinstance(engine, AsyncLLM)
|
||||
op = engine.output_processor
|
||||
request_ids = [
|
||||
*op.request_states.keys(),
|
||||
*op.parent_requests.keys(),
|
||||
]
|
||||
await engine.abort(request_ids, internal=True)
|
||||
return JSONResponse(
|
||||
content={"status": "aborted", "aborted": len(request_ids)},
|
||||
status_code=HTTPStatus.OK.value,
|
||||
)
|
||||
except Exception as err: # pragma: no cover - defensive
|
||||
logger.exception("Failed to abort requests")
|
||||
return JSONResponse(
|
||||
content={"error": f"Failed to abort requests: {err}"},
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/is_paused")
|
||||
async def is_paused(raw_request: Request) -> JSONResponse:
|
||||
"""Return the current pause status."""
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
paused = await engine.is_paused()
|
||||
except Exception as err: # pragma: no cover - defensive
|
||||
logger.exception("Failed to fetch pause status")
|
||||
return JSONResponse(
|
||||
content={"error": f"Failed to fetch pause status: {err}"},
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
)
|
||||
|
||||
return JSONResponse(content={"is_paused": paused})
|
||||
|
||||
|
||||
@router.post("/init_weight_transfer_engine")
|
||||
async def init_weight_transfer_engine(
|
||||
raw_request: Request,
|
||||
init_info: Annotated[dict[str, Any], Body(embed=True)],
|
||||
):
|
||||
with weight_operation_metrics().record("init"):
|
||||
await engine_client(raw_request).init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(init_info=init_info)
|
||||
)
|
||||
return JSONResponse(content={"message": "Weight transfer initialized"})
|
||||
|
||||
|
||||
@router.post("/shutdown_weight_transfer_engine")
|
||||
async def shutdown_weight_transfer_engine(raw_request: Request):
|
||||
with weight_operation_metrics().record("shutdown"):
|
||||
await engine_client(raw_request).shutdown_weight_transfer_engine()
|
||||
return JSONResponse(content={"message": "Weight transfer shut down"})
|
||||
|
||||
|
||||
@router.post("/start_weight_update")
|
||||
async def start_weight_update(raw_request: Request):
|
||||
with weight_operation_metrics().record("start"):
|
||||
await engine_client(raw_request).start_weight_update()
|
||||
return JSONResponse(content={"message": "Weight update started"})
|
||||
|
||||
|
||||
@router.post("/start_draft_weight_update")
|
||||
async def start_draft_weight_update(raw_request: Request):
|
||||
with weight_operation_metrics().record("start_draft"):
|
||||
await engine_client(raw_request).start_draft_weight_update()
|
||||
return JSONResponse(content={"message": "Draft weight update started"})
|
||||
|
||||
|
||||
@router.post("/update_weights")
|
||||
async def update_weights(
|
||||
raw_request: Request,
|
||||
update_info: Annotated[WeightTransferUpdatePayload, Body(embed=True)],
|
||||
):
|
||||
with weight_operation_metrics().record("update"):
|
||||
await engine_client(raw_request).update_weights(
|
||||
request=WeightTransferUpdateRequest(update_info=update_info)
|
||||
)
|
||||
return JSONResponse(content={"message": "Weights updated"})
|
||||
|
||||
|
||||
@router.post("/finish_weight_update")
|
||||
async def finish_weight_update(
|
||||
raw_request: Request,
|
||||
weight_version: Annotated[str | None, Body(embed=True)] = None,
|
||||
):
|
||||
with weight_operation_metrics().record("finish"):
|
||||
await engine_client(raw_request).finish_weight_update(weight_version)
|
||||
return JSONResponse(content={"message": "Weight update finished"})
|
||||
|
||||
|
||||
@router.post("/update_weight_version")
|
||||
async def update_weight_version(
|
||||
raw_request: Request,
|
||||
new_version: Annotated[str, Body(embed=True)],
|
||||
):
|
||||
await engine_client(raw_request).update_weight_version(new_version)
|
||||
return JSONResponse(content={"success": True, "new_version": new_version})
|
||||
|
||||
|
||||
@router.get("/weight_info")
|
||||
async def weight_info(raw_request: Request):
|
||||
weight_version = await engine_client(raw_request).get_weight_version()
|
||||
return JSONResponse(content={"weight_version": weight_version})
|
||||
|
||||
|
||||
@router.post("/weight_checker")
|
||||
async def weight_checker(
|
||||
raw_request: Request,
|
||||
action: Annotated[Literal["checksum", "reset", "compare"], Body(embed=True)],
|
||||
baseline: Annotated[dict[str, str] | None, Body(embed=True)] = None,
|
||||
) -> JSONResponse:
|
||||
"""Checksum, reset, or compare model weights against a baseline."""
|
||||
client = engine_client(raw_request)
|
||||
if action == "reset":
|
||||
await client.collective_rpc("reset_weights")
|
||||
return JSONResponse(content={"status": "reset"})
|
||||
if action == "compare" and baseline is None:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="action='compare' requires a 'baseline' object",
|
||||
)
|
||||
|
||||
checksums: dict[str, str] = {}
|
||||
for worker_checksums in await client.compute_weight_checksums():
|
||||
if duplicates := checksums.keys() & worker_checksums.keys():
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
detail=f"Duplicate weight checksum keys: {sorted(duplicates)}",
|
||||
)
|
||||
checksums.update(worker_checksums)
|
||||
if action == "checksum":
|
||||
return JSONResponse(content={"checksums": checksums})
|
||||
|
||||
assert baseline is not None
|
||||
mismatches = sorted(
|
||||
key
|
||||
for key in checksums.keys() | baseline.keys()
|
||||
if checksums.get(key) != baseline.get(key)
|
||||
)
|
||||
return JSONResponse(content={"match": not mismatches, "mismatches": mismatches})
|
||||
|
||||
|
||||
@router.get("/get_world_size")
|
||||
async def get_world_size(
|
||||
raw_request: Request,
|
||||
include_dp: bool = Query(True),
|
||||
):
|
||||
"""Get the world size from the parallel config.
|
||||
|
||||
Args:
|
||||
raw_request: The incoming FastAPI request, used to reach the engine
|
||||
client on the app state.
|
||||
include_dp: If True (default), returns the world size including
|
||||
data parallelism (TP * PP * DP). If False, returns the world
|
||||
size without data parallelism (TP * PP).
|
||||
|
||||
"""
|
||||
parallel_config = engine_client(raw_request).vllm_config.parallel_config
|
||||
if include_dp:
|
||||
world_size = parallel_config.world_size_across_dp
|
||||
else:
|
||||
world_size = parallel_config.world_size
|
||||
return JSONResponse(content={"world_size": world_size})
|
||||
@@ -1,56 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Duration and in-flight metrics for RL weight-transfer operations."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from threading import Lock
|
||||
from time import perf_counter
|
||||
from typing import Literal
|
||||
|
||||
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
|
||||
|
||||
Operation = Literal["init", "shutdown", "start", "start_draft", "update", "finish"]
|
||||
|
||||
|
||||
class WeightOperationMetrics:
|
||||
def __init__(self, registry: CollectorRegistry = REGISTRY):
|
||||
self.duration = Histogram(
|
||||
"vllm:rl_weight_update_operation_duration_seconds",
|
||||
"Duration of one weight-transfer operation.",
|
||||
["operation"],
|
||||
buckets=(0.01, 0.1, 1, 10, 30, 60, 120, 300, 600),
|
||||
registry=registry,
|
||||
)
|
||||
self.in_flight = Gauge(
|
||||
"vllm:rl_weight_update_operations_in_flight",
|
||||
"Weight-transfer operations currently awaited.",
|
||||
["operation"],
|
||||
multiprocess_mode="livesum",
|
||||
registry=registry,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def record(self, operation: Operation) -> Iterator[None]:
|
||||
in_flight = self.in_flight.labels(operation)
|
||||
in_flight.inc()
|
||||
started = perf_counter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
in_flight.dec()
|
||||
self.duration.labels(operation).observe(perf_counter() - started)
|
||||
|
||||
|
||||
_metrics: WeightOperationMetrics | None = None
|
||||
_metrics_lock = Lock()
|
||||
|
||||
|
||||
def weight_operation_metrics() -> WeightOperationMetrics:
|
||||
# Lazy, so the collectors exist only after multiprocess Prometheus setup.
|
||||
global _metrics
|
||||
if _metrics is None:
|
||||
with _metrics_lock:
|
||||
if _metrics is None:
|
||||
_metrics = WeightOperationMetrics()
|
||||
return _metrics
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequestUnion,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderChatStreamResponse,
|
||||
DerenderCompletionRequestUnion,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderCompletionStreamResponse,
|
||||
)
|
||||
from .serving import ServingDerender
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def derender(request: Request) -> ServingDerender | None:
|
||||
return getattr(request.app.state, "serving_derender", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/chat/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_chat_completion(
|
||||
request: DerenderChatRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a ChatCompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path; FastAPI validates and
|
||||
routes on the ``stream`` discriminator.
|
||||
|
||||
Non-streaming: body is ``DerenderChatRequest`` (``generate_response`` with
|
||||
the complete token list). Returns a ``ChatCompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderChatStreamRequest`` (one ``generate_chunk``
|
||||
delta + optional ``stream_state``). Returns a ``DerenderChatStreamResponse``
|
||||
(``chunk`` + ``stream_state``). The client carries ``stream_state`` between
|
||||
successive calls, one per SSE chunk from ``/inference/v1/generate``.
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Derender API"
|
||||
)
|
||||
|
||||
if isinstance(request, DerenderChatStreamRequest):
|
||||
stream_result = await handler.derender_chat_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_completion(
|
||||
request: DerenderCompletionRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a CompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path.
|
||||
|
||||
Non-streaming: body is ``DerenderCompletionRequest``. Returns a
|
||||
``CompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderCompletionStreamRequest`` (one
|
||||
``generate_chunk`` + optional ``stream_state``). Returns a
|
||||
``DerenderCompletionStreamResponse`` (``chunk`` + ``stream_state``).
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Derender API")
|
||||
|
||||
if isinstance(request, DerenderCompletionStreamRequest):
|
||||
stream_result = await handler.derender_completion_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderCompletionStreamResponse(
|
||||
chunk=chunk, stream_state=stream_state
|
||||
)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
@@ -1,434 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
|
||||
from ..token_in_token_out.mm_features import extract_mm_features
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderStreamState,
|
||||
GenerateTokensResponse,
|
||||
MultiModalFeatures,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingDerender(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||
online_derenderer: "OnlineDerenderer",
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=models.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.online_derenderer = online_derenderer
|
||||
|
||||
def _validate_derender_bounds(
|
||||
self,
|
||||
generate_responses: list[GenerateTokensResponse],
|
||||
) -> ErrorResponse | None:
|
||||
"""Reject derender payloads that exceed resource bounds.
|
||||
|
||||
Runs before any tokenizer.decode() or parser invocation to prevent
|
||||
CPU/memory exhaustion from oversized caller-supplied token structures.
|
||||
"""
|
||||
max_n = envs.VLLM_MAX_N_SEQUENCES
|
||||
max_model_len = self.model_config.max_model_len
|
||||
# See ModelConfig.max_logprobs for semantics and default value.
|
||||
max_logprobs = self.model_config.max_logprobs
|
||||
|
||||
if len(generate_responses) > max_n:
|
||||
return self.create_error_response(
|
||||
f"generate_responses count ({len(generate_responses)}) "
|
||||
f"exceeds server maximum ({max_n}). "
|
||||
f"Set VLLM_MAX_N_SEQUENCES to increase this limit."
|
||||
)
|
||||
|
||||
for gen in generate_responses:
|
||||
if len(gen.choices) > max_n:
|
||||
return self.create_error_response(
|
||||
f"choices count ({len(gen.choices)}) in response "
|
||||
f"'{gen.request_id}' exceeds server maximum ({max_n})."
|
||||
)
|
||||
|
||||
for choice in gen.choices:
|
||||
if choice.token_ids and len(choice.token_ids) > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"token_ids length ({len(choice.token_ids)}) in "
|
||||
f"choice {choice.index} exceeds "
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
if choice.logprobs and choice.logprobs.content:
|
||||
if len(choice.logprobs.content) > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"logprobs.content length "
|
||||
f"({len(choice.logprobs.content)}) in "
|
||||
f"choice {choice.index} exceeds "
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
for entry in choice.logprobs.content:
|
||||
if (
|
||||
max_logprobs >= 0
|
||||
and entry.top_logprobs
|
||||
and len(entry.top_logprobs) > max_logprobs
|
||||
):
|
||||
return self.create_error_response(
|
||||
f"top_logprobs count "
|
||||
f"({len(entry.top_logprobs)}) in "
|
||||
f"choice {choice.index} exceeds "
|
||||
f"max_logprobs ({max_logprobs})."
|
||||
)
|
||||
|
||||
if gen.prompt_logprobs and len(gen.prompt_logprobs) > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"prompt_logprobs length ({len(gen.prompt_logprobs)}) "
|
||||
f"in response '{gen.request_id}' exceeds "
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _validate_prompt_token_ids(
|
||||
self,
|
||||
prompt_token_ids: list[list[int] | None],
|
||||
) -> ErrorResponse | None:
|
||||
"""Reject caller-supplied prompt_token_ids longer than max_model_len."""
|
||||
max_model_len = self.model_config.max_model_len
|
||||
for ids in prompt_token_ids:
|
||||
if ids is not None and len(ids) > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"prompt_token_ids length ({len(ids)}) exceeds "
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
return None
|
||||
|
||||
async def derender_chat_response(
|
||||
self,
|
||||
request: DerenderChatRequest,
|
||||
) -> ChatCompletionResponse | ErrorResponse:
|
||||
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
|
||||
|
||||
Non-streaming only: expects the complete GenerateResponse with all
|
||||
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
|
||||
|
||||
When ``request.chat_request`` is provided, the parser splits the
|
||||
output into (reasoning, content, tool_calls). Otherwise falls
|
||||
back to plain detokenization.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
bounds_error = self._validate_derender_bounds([request.generate_response])
|
||||
if bounds_error is not None:
|
||||
return bounds_error
|
||||
|
||||
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
|
||||
if prompt_ids_error is not None:
|
||||
return prompt_ids_error
|
||||
|
||||
if self.online_derenderer.parser is not None and request.chat_request is None:
|
||||
return self.create_error_response(
|
||||
"chat_request is required when a tool or reasoning parser is "
|
||||
"configured because plain detokenization would leak raw parser "
|
||||
"markup into content."
|
||||
)
|
||||
|
||||
try:
|
||||
choices = await self.online_derenderer.derender_chat(
|
||||
request.generate_response,
|
||||
request.chat_request,
|
||||
request.prompt_token_ids,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
|
||||
prompt_tokens = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else 0
|
||||
)
|
||||
gen = request.generate_response
|
||||
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
logger.debug(
|
||||
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
|
||||
gen.request_id,
|
||||
model_name,
|
||||
len(choices),
|
||||
completion_tokens,
|
||||
)
|
||||
return ChatCompletionResponse(
|
||||
id=gen.request_id,
|
||||
model=model_name,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=gen.prompt_logprobs,
|
||||
kv_transfer_params=gen.kv_transfer_params,
|
||||
metrics=gen.metrics,
|
||||
)
|
||||
|
||||
async def derender_completion_response(
|
||||
self,
|
||||
request: DerenderCompletionRequest,
|
||||
) -> CompletionResponse | ErrorResponse:
|
||||
"""Postprocess a list of GenerateResponses into a CompletionResponse.
|
||||
|
||||
Non-streaming only. Mirrors the multi-prompt completions case: one
|
||||
GenerateResponse per prompt, parallel to the list[GenerateRequest]
|
||||
from /v1/completions/render.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
if not request.generate_responses:
|
||||
return self.create_error_response("generate_responses must not be empty")
|
||||
|
||||
bounds_error = self._validate_derender_bounds(request.generate_responses)
|
||||
if bounds_error is not None:
|
||||
return bounds_error
|
||||
|
||||
if request.prompt_token_ids is not None:
|
||||
prompt_ids_error = self._validate_prompt_token_ids(request.prompt_token_ids)
|
||||
if prompt_ids_error is not None:
|
||||
return prompt_ids_error
|
||||
|
||||
(
|
||||
choices,
|
||||
total_prompt_tokens,
|
||||
total_completion_tokens,
|
||||
) = await self.online_derenderer.derender_completion(
|
||||
request.generate_responses,
|
||||
request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
)
|
||||
|
||||
first = request.generate_responses[0]
|
||||
kv_params = first.kv_transfer_params
|
||||
if any(
|
||||
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
|
||||
):
|
||||
logger.warning(
|
||||
"derender_completion: kv_transfer_params differ across responses; "
|
||||
"setting to None on the aggregated response"
|
||||
)
|
||||
kv_params = None
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=total_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
logger.debug(
|
||||
"derender_completion request_id=%s model=%s choices=%d"
|
||||
" completion_tokens=%d",
|
||||
first.request_id,
|
||||
model_name,
|
||||
len(choices),
|
||||
total_completion_tokens,
|
||||
)
|
||||
return CompletionResponse(
|
||||
id=first.request_id,
|
||||
model=model_name,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
kv_transfer_params=kv_params,
|
||||
# Metrics describe one prompt. Parallel samples are already
|
||||
# suppressed by /generate; multi-prompt responses cannot be merged.
|
||||
metrics=first.metrics if len(request.generate_responses) == 1 else None,
|
||||
)
|
||||
|
||||
async def derender_chat_stream_response(
|
||||
self,
|
||||
request: DerenderChatStreamRequest,
|
||||
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState] | ErrorResponse:
|
||||
"""Streaming counterpart to ``derender_chat_response``.
|
||||
|
||||
Processes one ``GenerateStreamResponse`` chunk and returns the
|
||||
derendered chunk together with the updated client carried state.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
if self.online_derenderer.parser is not None and request.chat_request is None:
|
||||
return self.create_error_response(
|
||||
"chat_request is required when a tool or reasoning parser is "
|
||||
"configured because plain detokenization would leak raw parser "
|
||||
"markup into content."
|
||||
)
|
||||
|
||||
if (
|
||||
self.online_derenderer.parser is not None
|
||||
and request.prompt_token_ids is None
|
||||
):
|
||||
return self.create_error_response(
|
||||
"prompt_token_ids is required when a tool or reasoning "
|
||||
"parser is configured. Without it parse_delta cannot tell "
|
||||
"whether the prompt left reasoning open (e.g. a chat "
|
||||
"template that pre-opens <think>) and would misclassify "
|
||||
"reasoning content as plain content."
|
||||
)
|
||||
|
||||
# Each streamed chunk contains at most one choice per SSE event.
|
||||
# A single DerenderStreamState is threaded
|
||||
# through every choice in the chunk, so >1 would corrupt detok/parser
|
||||
# state across choices. See the matching check in
|
||||
# OnlineDerenderer.derender_chat_stream which this fails ahead of to
|
||||
# avoid touching the tokenizer at all on a malformed chunk.
|
||||
num_choices = len(request.generate_chunk.choices)
|
||||
if num_choices > 1:
|
||||
return self.create_error_response(
|
||||
f"derender_chat_stream expects at most one choice per chunk "
|
||||
f"(got {num_choices})."
|
||||
)
|
||||
|
||||
# prompt_token_ids is folded in here too. It is caller supplied and
|
||||
# otherwise unbounded. A parser configured deployment rescans it
|
||||
# in full on every chunk (is_reasoning_end /
|
||||
# adjust_initial_state_from_prompt, see abstract_parser.parse_delta)
|
||||
# since parser state cannot be carried across calls.
|
||||
prompt_len = (
|
||||
len(request.prompt_token_ids) if request.prompt_token_ids is not None else 0
|
||||
)
|
||||
prior_len = (
|
||||
len(request.stream_state.output_token_ids)
|
||||
if request.stream_state is not None
|
||||
else 0
|
||||
)
|
||||
delta_len = sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
)
|
||||
max_model_len = self.model_config.max_model_len
|
||||
if prompt_len + prior_len + delta_len > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"prompt_token_ids length ({prompt_len}) plus "
|
||||
f"output_token_ids length ({prior_len}) plus delta "
|
||||
f"({delta_len}) exceeds max_model_len ({max_model_len})."
|
||||
)
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
try:
|
||||
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
|
||||
model=model_name,
|
||||
generate_chunk=request.generate_chunk,
|
||||
state=request.stream_state,
|
||||
chat_request=request.chat_request,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
)
|
||||
except Exception as exc:
|
||||
# The two ValueErrors derender_chat_stream can raise directly
|
||||
# (missing chat_request, >1 choice per chunk) are already
|
||||
# pre-checked above, so anything reaching here comes from the
|
||||
# parser replaying entirely client controlled state. Hermes
|
||||
# swallows its own streaming exceptions but finalize_generation
|
||||
# and engine based parsers don't, so an unexpected parser
|
||||
# failure on malformed (but well typed) state must surface as
|
||||
# a 400 here rather than an unhandled 500.
|
||||
logger.debug(
|
||||
"derender stream replay failed: %s: %s", type(exc).__name__, exc
|
||||
)
|
||||
return self.create_error_response("invalid stream_state: derender failed")
|
||||
|
||||
logger.debug(
|
||||
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
|
||||
request.generate_chunk.request_id,
|
||||
model_name,
|
||||
sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
),
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
async def derender_completion_stream_response(
|
||||
self,
|
||||
request: DerenderCompletionStreamRequest,
|
||||
) -> tuple[CompletionStreamResponse, DerenderStreamState] | ErrorResponse:
|
||||
"""Streaming counterpart to ``derender_completion_response``.
|
||||
|
||||
Processes one ``GenerateStreamResponse`` chunk (one output sequence's
|
||||
delta) and returns the derendered chunk and updated state.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
|
||||
if prompt_ids_error is not None:
|
||||
return prompt_ids_error
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
try:
|
||||
(
|
||||
chunk,
|
||||
updated_state,
|
||||
) = await self.online_derenderer.derender_completion_stream(
|
||||
model=model_name,
|
||||
generate_chunk=request.generate_chunk,
|
||||
state=request.stream_state,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
except (KeyError, IndexError) as exc:
|
||||
return self.create_error_response(
|
||||
f"invalid stream_state: detokenization failed ({exc})"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_completion_stream request_id=%s model=%s delta_tokens=%d",
|
||||
request.generate_chunk.request_id,
|
||||
model_name,
|
||||
sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
),
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
@staticmethod
|
||||
def _extract_mm_features(
|
||||
engine_input: EngineInput,
|
||||
) -> MultiModalFeatures | None:
|
||||
return extract_mm_features(engine_input)
|
||||
@@ -25,35 +25,12 @@ else:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def init_render_state(
|
||||
state: "State",
|
||||
request_logger: RequestLogger | None,
|
||||
):
|
||||
from .derender.serving import ServingDerender
|
||||
from .render.serving import ServingRender
|
||||
|
||||
state.serving_render = ServingRender(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
tool_server=state.tool_server,
|
||||
)
|
||||
|
||||
state.serving_derender = ServingDerender(
|
||||
state.openai_serving_models,
|
||||
state.online_derenderer,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
|
||||
def init_scale_out_state(
|
||||
state: "State",
|
||||
args: "Namespace",
|
||||
engine_client: "EngineClient",
|
||||
request_logger: RequestLogger | None,
|
||||
):
|
||||
init_render_state(state, request_logger)
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
|
||||
state.serving_tokens = ServingTokens(
|
||||
@@ -72,14 +49,10 @@ def register_scale_out_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
from .derender.api_router import router as derender_render
|
||||
from .render.api_router import router as render_render
|
||||
|
||||
args = getattr(app.state, "args", None)
|
||||
# `vllm launch render` and `vllm serve --tokens-only` are dedicated to
|
||||
# serving these endpoints, so they always register them regardless of
|
||||
# `--enable-scale-out`.
|
||||
dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False)
|
||||
# `vllm serve --tokens-only` is dedicated to serving these endpoints, so it
|
||||
# always registers them regardless of `--enable-scale-out`.
|
||||
dedicated_mode = getattr(args, "tokens_only", False)
|
||||
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
|
||||
if not enabled:
|
||||
message = (
|
||||
@@ -96,11 +69,7 @@ def register_scale_out_api_routers(
|
||||
|
||||
from .token_in_token_out.api_router import router as token_router
|
||||
|
||||
for route in (
|
||||
*render_render.routes,
|
||||
*derender_render.routes,
|
||||
*token_router.routes,
|
||||
):
|
||||
for route in token_router.routes:
|
||||
if isinstance(route, APIRoute):
|
||||
app.add_api_route(
|
||||
route.path,
|
||||
@@ -110,9 +79,6 @@ def register_scale_out_api_routers(
|
||||
)
|
||||
return
|
||||
|
||||
app.include_router(render_render)
|
||||
app.include_router(derender_render)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from .token_in_token_out.api_router import (
|
||||
attach_router as attach_disagg_router,
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..token_in_token_out.protocol import GenerateRequest
|
||||
from .serving import ServingRender
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def render(request: Request) -> ServingRender | None:
|
||||
return getattr(request.app.state, "serving_render", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/chat/completions/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Render API"
|
||||
)
|
||||
|
||||
result = await handler.render_chat_request(request, raw_request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/messages/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_messages(request: AnthropicMessagesRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Messages Render API")
|
||||
|
||||
result = await handler.render_messages_request(request, raw_request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=list[GenerateRequest],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_completion(request: CompletionRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Render API")
|
||||
|
||||
result = await handler.render_completion_request(request, raw_request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=[item.model_dump() for item in result])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/responses/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_responses(request: ResponsesRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Responses Render API")
|
||||
|
||||
result = await handler.render_responses_request(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
@@ -1,385 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.generate.base.protocol import MMPayload
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||
extract_mm_features,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
ReasoningParserKwargs,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.parse import MultiModalDataParser
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
extract_prompt_len,
|
||||
)
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
|
||||
|
||||
def _hoist_transfer_params(
|
||||
params: SamplingParams,
|
||||
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
||||
"""Move KV/EC transfer params out of `extra_args`.
|
||||
|
||||
Render emits them only as top-level `GenerateRequest` fields. Generate
|
||||
copies them back into `extra_args` for the engine.
|
||||
"""
|
||||
if not params.extra_args:
|
||||
return None, None
|
||||
# to_sampling_params aliases request.vllm_xargs, so copy before popping.
|
||||
extra = dict(params.extra_args)
|
||||
kv = extra.pop("kv_transfer_params", None)
|
||||
ec = extra.pop("ec_transfer_params", None)
|
||||
params.extra_args = extra or None
|
||||
return kv, ec
|
||||
|
||||
|
||||
class ServingRender(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||
online_renderer: "OnlineRenderer",
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
tool_server: "ToolServer | None" = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=online_renderer.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.online_renderer = online_renderer
|
||||
self.tool_server = tool_server
|
||||
|
||||
self._merge_inline_system = (
|
||||
AnthropicServingMessages._should_merge_inline_system(online_renderer)
|
||||
)
|
||||
|
||||
self._placeholder_metadata_parser: MultiModalDataParser | None = None
|
||||
self._placeholder_metadata_parser_failed = False
|
||||
|
||||
self.default_sampling_params = (
|
||||
online_renderer.model_config.get_diff_sampling_param()
|
||||
)
|
||||
mc = online_renderer.model_config
|
||||
self.override_max_tokens = (
|
||||
self.default_sampling_params.get("max_tokens")
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
|
||||
async def render_chat_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> GenerateRequest | ErrorResponse:
|
||||
"""Validate the model and preprocess a chat completion request.
|
||||
|
||||
This is the authoritative implementation used directly by the
|
||||
GPU-less render server and delegated to by OpenAIServingChat.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
if request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Beam search is not supported by the render endpoint"
|
||||
)
|
||||
|
||||
result = await self.online_renderer.render_chat(request, skip_mm_cache=True)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
_, engine_inputs = result
|
||||
|
||||
if len(engine_inputs) != 1:
|
||||
return self.create_error_response(
|
||||
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
|
||||
)
|
||||
|
||||
engine_input = engine_inputs[0]
|
||||
|
||||
prompt_components = extract_prompt_components(self.model_config, engine_input)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
token_ids = list(token_ids)
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_completion_tokens
|
||||
if request.max_completion_tokens is not None
|
||||
else request.max_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
|
||||
|
||||
# Resolve here what OpenAIServingChat passes to the engine:
|
||||
# `_grammar_from_parser` (set by `adjust_request`) is not serialized.
|
||||
reasoning_parser_kwargs = self._reasoning_parser_kwargs(request)
|
||||
parser = None
|
||||
if reasoning_parser_kwargs is not None:
|
||||
assert self.online_renderer.parser is not None
|
||||
parser = self.online_renderer.parser(
|
||||
self.online_renderer.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
chat_template_kwargs=reasoning_parser_kwargs.chat_template_kwargs,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
reasoning_ended = request.resolve_reasoning_ended(parser, token_ids)
|
||||
|
||||
request_id = (
|
||||
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||
)
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
features=self._extract_mm_features(
|
||||
engine_input, mm_payload=request.mm_payload
|
||||
),
|
||||
sampling_params=params,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
model=request.model,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
async def render_messages_request(
|
||||
self,
|
||||
request: AnthropicMessagesRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> GenerateRequest | ErrorResponse:
|
||||
"""Validate the model and preprocess an Anthropic Messages request.
|
||||
|
||||
Converts the request to the OpenAI chat format using the same
|
||||
conversion as the /v1/messages server path, then delegates to
|
||||
render_chat_request so the rendered tokens match the server exactly.
|
||||
"""
|
||||
chat_req = AnthropicServingMessages.to_chat_completion_request(
|
||||
request, merge_inline_system=self._merge_inline_system
|
||||
)
|
||||
chat_req.mm_payload = request.mm_payload
|
||||
return await self.render_chat_request(chat_req, raw_request)
|
||||
|
||||
async def render_completion_request(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> list[GenerateRequest] | ErrorResponse:
|
||||
"""Validate the model and preprocess a completion request.
|
||||
|
||||
This is the authoritative implementation used directly by the
|
||||
GPU-less render server and delegated to by OpenAIServingCompletion.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
if request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Beam search is not supported by the render endpoint"
|
||||
)
|
||||
|
||||
result = await self.online_renderer.render_completion(
|
||||
request, skip_mm_cache=True
|
||||
)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
base_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||
generate_requests: list[GenerateRequest] = []
|
||||
for i, engine_input in enumerate(result):
|
||||
prompt_components = extract_prompt_components(
|
||||
self.model_config, engine_input
|
||||
)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
token_ids = list(token_ids)
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
)
|
||||
params = request.to_sampling_params(
|
||||
max_tokens, self.default_sampling_params
|
||||
)
|
||||
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
|
||||
|
||||
request_id = base_id if len(result) == 1 else f"{base_id}-{i}"
|
||||
|
||||
generate_requests.append(
|
||||
GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
features=self._extract_mm_features(
|
||||
engine_input, mm_payload=request.mm_payload
|
||||
),
|
||||
sampling_params=params,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
model=request.model,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
)
|
||||
|
||||
return generate_requests
|
||||
|
||||
async def render_responses_request(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
) -> GenerateRequest | ErrorResponse:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
if request.previous_response_id is not None:
|
||||
return self.create_error_response(
|
||||
message=(
|
||||
"previous_response_id is not supported by the stateless "
|
||||
"render endpoint."
|
||||
),
|
||||
err_type="invalid_request_error",
|
||||
param="previous_response_id",
|
||||
)
|
||||
|
||||
result = await self.online_renderer.render_responses(
|
||||
request,
|
||||
previous_messages=None,
|
||||
previous_response_outputs=None,
|
||||
tool_server=self.tool_server,
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
engine_input = result.engine_input
|
||||
prompt_components = extract_prompt_components(self.model_config, engine_input)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_output_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request.request_id,
|
||||
token_ids=list(token_ids),
|
||||
features=self._extract_mm_features(
|
||||
engine_input, mm_payload=request.mm_payload
|
||||
),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
|
||||
stream=bool(request.stream),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
def _reasoning_parser_kwargs(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ReasoningParserKwargs | None:
|
||||
parser_cls = self.online_renderer.parser
|
||||
if parser_cls is None or parser_cls.reasoning_parser_cls is None:
|
||||
return None
|
||||
return ReasoningParserKwargs(
|
||||
chat_template_kwargs=self.online_renderer.effective_chat_template_kwargs(
|
||||
request
|
||||
)
|
||||
)
|
||||
|
||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||
"""Fields the EC consumer still requires after embeddings transfer."""
|
||||
if self._placeholder_metadata_parser_failed:
|
||||
return set()
|
||||
if self._placeholder_metadata_parser is None:
|
||||
try:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
self._placeholder_metadata_parser = (
|
||||
MULTIMODAL_REGISTRY.create_processor(
|
||||
self.model_config
|
||||
).info.data_parser
|
||||
)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not load placeholder metadata fields; "
|
||||
"mm_metadata will use keep_on_cpu fields only."
|
||||
)
|
||||
self._placeholder_metadata_parser_failed = True
|
||||
return set()
|
||||
return set(
|
||||
self._placeholder_metadata_parser.placeholder_metadata_fields(modality)
|
||||
)
|
||||
|
||||
def _extract_mm_features(
|
||||
self,
|
||||
engine_input: EngineInput,
|
||||
*,
|
||||
mm_payload: MMPayload = "tensors",
|
||||
) -> MultiModalFeatures | None:
|
||||
return extract_mm_features(
|
||||
engine_input,
|
||||
metadata_fields_for=self._placeholder_metadata_fields,
|
||||
mm_payload=mm_payload,
|
||||
)
|
||||
@@ -108,8 +108,8 @@ def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
if not getattr(app.state.args, "tokens_only", False):
|
||||
return
|
||||
# The RLHF dev router registers its own /abort_requests first. Registering
|
||||
# this one too would only add a shadowed route with a duplicate operation ID.
|
||||
# An endpoint plugin may already serve /abort_requests. Registering this
|
||||
# one too would only add a shadowed route with a duplicate operation ID.
|
||||
has_abort_route = any(
|
||||
getattr(route, "path", None) == "/abort_requests" for route in app.routes
|
||||
)
|
||||
|
||||
@@ -685,7 +685,6 @@ class DerenderStreamState(BaseModel):
|
||||
entire accumulated text on each invocation, `parse_delta()` itself is
|
||||
O(n) yielding a true worst case compute cost of O(n³) per generation.
|
||||
No caching is performed. Work is bounded by `max_model_len`.
|
||||
See `OnlineDerenderer._derender_chat_stream_parsed`.
|
||||
|
||||
The detokenization strategy carries the incremental decode offsets
|
||||
directly rather than re-sending the whole token history each chunk.
|
||||
@@ -823,7 +822,7 @@ class DerenderChatStreamRequest(BaseModel):
|
||||
`<think>`). `prompt_tokens` is a usage count and cannot serve this
|
||||
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
|
||||
|
||||
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
|
||||
Rejected with a 400 when a tool or reasoning
|
||||
parser is configured and this is omitted. Without it, `parse_delta`
|
||||
cannot tell whether the prompt left reasoning open and would silently
|
||||
misclassify reasoning content as plain content. On all paths it also
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import ExternalElasticEPScalePhase
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/scale_elastic_ep",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"model": dict},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.REQUEST_TIMEOUT.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def scale_elastic_ep(raw_request: Request):
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(status_code=400, detail="Invalid JSON format") from e
|
||||
|
||||
new_data_parallel_size = body.get("new_data_parallel_size")
|
||||
drain_timeout = body.get("drain_timeout", 120) # Default 2 minutes
|
||||
|
||||
if new_data_parallel_size is None:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="new_data_parallel_size is required"
|
||||
)
|
||||
|
||||
if not isinstance(new_data_parallel_size, int) or new_data_parallel_size <= 0:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="new_data_parallel_size must be a positive integer",
|
||||
)
|
||||
|
||||
if not isinstance(drain_timeout, int) or drain_timeout <= 0:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="drain_timeout must be a positive integer"
|
||||
)
|
||||
|
||||
client = engine_client(raw_request)
|
||||
try:
|
||||
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
|
||||
return JSONResponse(
|
||||
{
|
||||
"message": f"Scaled to {new_data_parallel_size} data parallel engines",
|
||||
}
|
||||
)
|
||||
except TimeoutError as e:
|
||||
raise HTTPException(
|
||||
status_code=408,
|
||||
detail="Scale failed due to request drain timeout "
|
||||
f"after {drain_timeout} seconds",
|
||||
) from e
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except Exception as e:
|
||||
logger.error("Scale failed: %s", e)
|
||||
raise HTTPException(status_code=500, detail="Scale failed") from e
|
||||
|
||||
|
||||
@router.post("/is_scaling_elastic_ep")
|
||||
async def is_scaling_elastic_ep(raw_request: Request):
|
||||
# External operation status comes from the shared store. Middleware gating
|
||||
# remains process-local, so scaling requests must reach every old-rank API.
|
||||
try:
|
||||
status = await engine_client(raw_request).get_external_elastic_ep_status()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to query external Elastic EP status: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="External Elastic EP status is temporarily unavailable",
|
||||
) from e
|
||||
if status is None:
|
||||
# Non-external EEP modes retain the process-local middleware state.
|
||||
is_scaling = get_scaling_elastic_ep()
|
||||
status = {
|
||||
"phase": (
|
||||
ExternalElasticEPScalePhase.COMMITTING.value
|
||||
if is_scaling
|
||||
else ExternalElasticEPScalePhase.IDLE.value
|
||||
),
|
||||
"epoch": None,
|
||||
"error": None,
|
||||
"requested_data_parallel_size": None,
|
||||
}
|
||||
else:
|
||||
is_scaling = status["phase"] in (
|
||||
ExternalElasticEPScalePhase.PREPARING.value,
|
||||
ExternalElasticEPScalePhase.COMMITTING.value,
|
||||
)
|
||||
return JSONResponse({"is_scaling_elastic_ep": is_scaling, **status})
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -16,7 +16,3 @@ def register_instrumentator_api_routers(app: FastAPI):
|
||||
from .metrics import attach_router as metrics_attach_router
|
||||
|
||||
metrics_attach_router(app)
|
||||
|
||||
from .offline_docs import attach_router as offline_docs_attach_router
|
||||
|
||||
offline_docs_attach_router(app)
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline FastAPI documentation support for air-gapped environments."""
|
||||
|
||||
import pathlib
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.openapi.docs import (
|
||||
get_swagger_ui_html,
|
||||
get_swagger_ui_oauth2_redirect_html,
|
||||
)
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Attach offline docs router if enabled via args."""
|
||||
args = getattr(app.state, "args", None)
|
||||
if args is None or not getattr(args, "enable_offline_docs", False):
|
||||
return
|
||||
|
||||
static_dir = pathlib.Path(__file__).parent / "static"
|
||||
|
||||
if not static_dir.exists():
|
||||
logger.warning(
|
||||
"Static directory not found at %s. Offline docs will not be available.",
|
||||
static_dir,
|
||||
)
|
||||
return
|
||||
|
||||
app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
|
||||
|
||||
@app.get("/docs", include_in_schema=False)
|
||||
async def custom_swagger_ui_html():
|
||||
return get_swagger_ui_html(
|
||||
openapi_url=app.openapi_url,
|
||||
title=app.title + " - Swagger UI",
|
||||
oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
|
||||
swagger_js_url="/static/swagger-ui-bundle.js",
|
||||
swagger_css_url="/static/swagger-ui.css",
|
||||
)
|
||||
|
||||
@app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False)
|
||||
async def swagger_ui_redirect():
|
||||
return get_swagger_ui_oauth2_redirect_html()
|
||||
|
||||
logger.info("Offline documentation enabled with vendored static assets")
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -8,7 +8,7 @@ from starlette.datastructures import Headers
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""SageMaker-specific integration for vLLM."""
|
||||
@@ -1,156 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
import pydantic
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.instrumentator.basic import base
|
||||
from vllm.entrypoints.serve.instrumentator.health import health
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
|
||||
# (requires typing_extensions >= 4.13)
|
||||
RequestType = Any
|
||||
GetHandlerFn = Callable[[Request], BaseServing | None]
|
||||
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
|
||||
|
||||
|
||||
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
|
||||
"""Snapshot handler levels for loggers that may be affected by third-party
|
||||
logging configuration side effects (e.g. model_hosting_container_standards
|
||||
calling configure_root_logger() at import time)."""
|
||||
loggers = [logging.getLogger(), logging.getLogger("vllm")]
|
||||
seen: set[int] = set()
|
||||
snapshot: list[tuple[logging.Handler, int]] = []
|
||||
for logger in loggers:
|
||||
for handler in logger.handlers:
|
||||
if id(handler) not in seen:
|
||||
seen.add(id(handler))
|
||||
snapshot.append((handler, handler.level))
|
||||
return snapshot
|
||||
|
||||
|
||||
def _restore_handler_levels(
|
||||
snapshot: list[tuple[logging.Handler, int]],
|
||||
) -> None:
|
||||
"""Restore handler levels from a snapshot."""
|
||||
for handler, level in snapshot:
|
||||
handler.setLevel(level)
|
||||
|
||||
|
||||
def attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
"""Attach the SageMaker hosting endpoints to the API server.
|
||||
|
||||
Handler levels are snapshotted and restored because importing
|
||||
model_hosting_container_standards may reconfigure root logging.
|
||||
"""
|
||||
snapshot = _snapshot_handler_levels()
|
||||
try:
|
||||
_attach_router(app, supported_tasks, model_config)
|
||||
finally:
|
||||
_restore_handler_levels(snapshot)
|
||||
|
||||
|
||||
def _attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
"""Register the SageMaker hosting routes (/ping, /invocations) on the
|
||||
app."""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# NOTE: Construct the TypeAdapters only once
|
||||
INVOCATION_TYPES = get_generate_invocation_types(supported_tasks, model_config)
|
||||
|
||||
INVOCATION_VALIDATORS = [
|
||||
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
|
||||
for request_type, (get_handler, endpoint) in INVOCATION_TYPES
|
||||
]
|
||||
|
||||
@router.post("/ping", response_class=Response)
|
||||
@router.get("/ping", response_class=Response)
|
||||
@sagemaker_standards.register_ping_handler
|
||||
async def ping(raw_request: Request) -> Response:
|
||||
"""Ping check. Endpoint required for SageMaker."""
|
||||
return await health(raw_request)
|
||||
|
||||
@router.post(
|
||||
"/invocations",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@sagemaker_standards.register_invocation_handler
|
||||
@sagemaker_standards.stateful_session_manager()
|
||||
@sagemaker_standards.inject_adapter_id(adapter_path="model")
|
||||
async def invocations(raw_request: Request):
|
||||
"""For SageMaker, routes requests based on the request type."""
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=f"JSON decode error: {e}",
|
||||
) from e
|
||||
|
||||
valid_endpoints = [
|
||||
(validator, endpoint)
|
||||
for validator, (get_handler, endpoint) in INVOCATION_VALIDATORS
|
||||
if get_handler(raw_request) is not None
|
||||
]
|
||||
|
||||
for request_validator, endpoint in valid_endpoints:
|
||||
try:
|
||||
request = request_validator.validate_python(body)
|
||||
except pydantic.ValidationError:
|
||||
continue
|
||||
|
||||
return await endpoint(request, raw_request)
|
||||
|
||||
type_names = [
|
||||
t.__name__ if isinstance(t := validator._type, type) else str(t)
|
||||
for validator, _ in valid_endpoints
|
||||
]
|
||||
msg = f"Cannot find suitable handler for request. Expected one of: {type_names}"
|
||||
res = base(raw_request).create_error_response(message=msg)
|
||||
return JSONResponse(content=res.model_dump(), status_code=res.error.code)
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
|
||||
"""Bootstrap the app with the SageMaker hosting standards.
|
||||
|
||||
Handler levels are restored right after the import because importing
|
||||
model_hosting_container_standards may reconfigure root logging, and
|
||||
bootstrap must run with the original levels.
|
||||
"""
|
||||
snapshot = _snapshot_handler_levels()
|
||||
try:
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
app = sagemaker_standards.bootstrap(app)
|
||||
finally:
|
||||
_restore_handler_levels(snapshot)
|
||||
return app
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ASGI middleware for WebSocket Prometheus metrics.
|
||||
|
||||
Modeled after prometheus-fastapi-instrumentator, this middleware
|
||||
transparently instruments WebSocket endpoints with standard metrics
|
||||
without requiring changes to handler code.
|
||||
|
||||
NOTE: This module intentionally has zero vllm imports so that it can
|
||||
be extracted into a standalone package (similar to
|
||||
prometheus-fastapi-instrumentator) in the future. Please keep it that way.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Awaitable
|
||||
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
# Standard WebSocket metric names (not vllm-specific, following
|
||||
# the same convention as prometheus-fastapi-instrumentator).
|
||||
_active_sessions = Gauge(
|
||||
name="vllm:websocket_connections_active",
|
||||
documentation="Number of currently active WebSocket connections.",
|
||||
multiprocess_mode="livesum",
|
||||
)
|
||||
|
||||
_total_sessions = Counter(
|
||||
name="vllm:websocket_connections_total",
|
||||
documentation="Total number of WebSocket connections.",
|
||||
)
|
||||
|
||||
_session_duration = Histogram(
|
||||
name="vllm:websocket_connection_duration_seconds",
|
||||
documentation="Duration of WebSocket connections in seconds.",
|
||||
buckets=[0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1800],
|
||||
)
|
||||
|
||||
|
||||
class WebSocketMetricsMiddleware:
|
||||
"""Pure ASGI middleware that instruments WebSocket connections.
|
||||
|
||||
Tracks active connections (gauge), total connections (counter),
|
||||
and connection duration (histogram) for all WebSocket endpoints.
|
||||
|
||||
Usage::
|
||||
|
||||
app.add_middleware(WebSocketMetricsMiddleware)
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
|
||||
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
|
||||
if scope["type"] != "websocket":
|
||||
return self.app(scope, receive, send)
|
||||
|
||||
return self._handle_websocket(scope, receive, send)
|
||||
|
||||
async def _handle_websocket(
|
||||
self, scope: Scope, receive: Receive, send: Send
|
||||
) -> None:
|
||||
start_time: float | None = None
|
||||
|
||||
async def send_wrapper(message: Message) -> None:
|
||||
nonlocal start_time
|
||||
if message["type"] == "websocket.accept":
|
||||
start_time = time.monotonic()
|
||||
_active_sessions.inc()
|
||||
_total_sessions.inc()
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, send_wrapper)
|
||||
finally:
|
||||
if start_time is not None:
|
||||
_active_sessions.dec()
|
||||
_session_duration.observe(time.monotonic() - start_time)
|
||||
-151
@@ -49,15 +49,10 @@ if TYPE_CHECKING:
|
||||
VLLM_TRACE_FUNCTION: int = 0
|
||||
VLLM_USE_FLASHINFER_SAMPLER: bool = True
|
||||
VLLM_PP_LAYER_PARTITION: str | None = None
|
||||
VLLM_CPU_KVCACHE_SPACE: int | None = 0
|
||||
VLLM_CPU_OMP_THREADS_BIND: str = "auto"
|
||||
VLLM_CPU_NUM_OF_RESERVED_CPU: int | None = None
|
||||
VLLM_CPU_CI_ENV: bool = False
|
||||
VLLM_CPU_ATTN_SPLIT_KV: bool = True
|
||||
VLLM_ZENTORCH_WEIGHT_PREPACK: bool = True
|
||||
VLLM_CPU_INT4_W4A8: bool = True
|
||||
VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
|
||||
VLLM_XLA_CHECK_RECOMPILATION: bool = False
|
||||
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: int = 512
|
||||
VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN: int = 8192
|
||||
VLLM_XPU_PP_MICROBATCH: bool = True
|
||||
@@ -97,10 +92,6 @@ if TYPE_CHECKING:
|
||||
VLLM_USE_PRECOMPILED: bool = False
|
||||
VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
|
||||
VLLM_DOCKER_BUILD_CONTEXT: bool = False
|
||||
VLLM_BUILD_COMMIT: str = "unknown"
|
||||
VLLM_BUILD_PIPELINE: str = "local"
|
||||
VLLM_BUILD_URL: str = ""
|
||||
VLLM_IMAGE_TAG: str = ""
|
||||
VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
|
||||
CMAKE_BUILD_TYPE: Literal["Debug", "Release", "RelWithDebInfo"] | None = None
|
||||
VERBOSE: bool = False
|
||||
@@ -125,7 +116,6 @@ if TYPE_CHECKING:
|
||||
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
|
||||
VLLM_SKIP_P2P_CHECK: bool = False
|
||||
VLLM_DISABLED_KERNELS: list[str] = []
|
||||
VLLM_USE_HW_AGNOSTIC: bool = False
|
||||
VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True
|
||||
VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda"
|
||||
VLLM_DISABLE_PYNCCL: bool = False
|
||||
@@ -142,8 +132,6 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
|
||||
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["auto", "asm", "segmented"] = "auto"
|
||||
VLLM_ROCM_USE_AITER_MHA: bool = True
|
||||
VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: bool = False
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
|
||||
@@ -160,12 +148,10 @@ if TYPE_CHECKING:
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
|
||||
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
|
||||
VLLM_DISABLE_COMPILE_CACHE: bool = False
|
||||
VLLM_REPLICATE_EMBED: bool = False
|
||||
VLLM_USE_LAYERNAME: bool = True
|
||||
VLLM_SERVER_DEV_MODE: bool = False
|
||||
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
|
||||
VLLM_MLA_DISABLE: bool = False
|
||||
VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = True
|
||||
VLLM_RAY_PER_WORKER_GPUS: float = 1.0
|
||||
VLLM_RAY_BUNDLE_INDICES: str = ""
|
||||
VLLM_CUDART_SO_PATH: str | None = None
|
||||
@@ -191,12 +177,10 @@ if TYPE_CHECKING:
|
||||
VLLM_B12X_MOE_FP4_FORCE_A16: bool = False
|
||||
VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
|
||||
VLLM_V1_USE_OUTLINES_CACHE: bool = False
|
||||
VLLM_TPU_USING_PATHWAYS: bool = False
|
||||
VLLM_USE_DEEP_GEMM: bool = True
|
||||
VLLM_MOE_USE_DEEP_GEMM: bool = True
|
||||
VLLM_USE_DEEP_GEMM_E8M0: bool = True
|
||||
VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES: bool = True
|
||||
VLLM_DCP_Q_REPLICATE: bool = False
|
||||
VLLM_USE_DIRECT_DCP_A2A: bool | None = None
|
||||
VLLM_USE_DIRECT_DCP_Q_GATHER: bool | None = None
|
||||
VLLM_USE_DIRECT_DCP_KV_GATHER: bool | None = None
|
||||
@@ -207,10 +191,6 @@ if TYPE_CHECKING:
|
||||
] = "relax"
|
||||
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
|
||||
VLLM_MOE_SKIP_PADDING: bool = True
|
||||
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
|
||||
VLLM_KIMI_K3_AUX_ATTN_RES_STREAM: bool = False
|
||||
VLLM_KIMI_K3_GEMM_AR: bool = True
|
||||
VLLM_ENABLE_GEMM_RS: bool = False
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
|
||||
@@ -224,25 +204,15 @@ if TYPE_CHECKING:
|
||||
VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False
|
||||
VLLM_ABLITERATION: str = ""
|
||||
VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
|
||||
VLLM_P2P_SIDE_CHANNEL_HOST: str = "localhost"
|
||||
VLLM_P2P_SIDE_CHANNEL_PORT: int = 5710
|
||||
VLLM_P2P_MAX_PEERS: int = 128
|
||||
VLLM_P2P_HANDSHAKE_TIMEOUT_S: int = 30
|
||||
VLLM_P2P_IDLE_TIMEOUT_S: int = 300
|
||||
VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost"
|
||||
VLLM_EC_SIDE_CHANNEL_PORT: int = 5601
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
|
||||
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
|
||||
VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1
|
||||
VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9
|
||||
MOONCAKE_PREFERRED_SEGMENT: str | None = None
|
||||
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
|
||||
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
|
||||
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
|
||||
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
|
||||
VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT: bool = True
|
||||
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
|
||||
VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
|
||||
@@ -260,13 +230,10 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: int | None = None
|
||||
VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None
|
||||
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None
|
||||
VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480
|
||||
VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0
|
||||
VLLM_ENABLE_CUDAGRAPH_GC: bool = False
|
||||
VLLM_LOOPBACK_IP: str = ""
|
||||
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
|
||||
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
|
||||
VLLM_ENABLE_COHERE_API: bool = False
|
||||
VLLM_HAS_FLASHINFER_CUBIN: bool = False
|
||||
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
|
||||
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
|
||||
@@ -299,9 +266,7 @@ if TYPE_CHECKING:
|
||||
VLLM_GC_DEBUG: str = ""
|
||||
VLLM_DEBUG_WORKSPACE: bool = False
|
||||
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
|
||||
VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION: bool = False
|
||||
VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
|
||||
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
|
||||
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
|
||||
VLLM_USE_V2_MODEL_RUNNER: bool | None = None
|
||||
VLLM_LOG_MODEL_INSPECTION: bool = False
|
||||
@@ -323,11 +288,9 @@ if TYPE_CHECKING:
|
||||
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True
|
||||
VLLM_XPU_FORCE_N_CONTIG_WEIGHT: bool = False
|
||||
VLLM_XPU_USE_SAMPLER_KERNEL: bool = True
|
||||
VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto"
|
||||
VLLM_LORA_ENABLE_DUAL_STREAM: bool = False
|
||||
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
|
||||
VLLM_NIC_SELECTION_VARS: str = ""
|
||||
VLLM_ENABLE_HPC_OPS: bool = False
|
||||
VLLM_REQUEST_BODY_LOG_LEVEL: str = "DEBUG"
|
||||
|
||||
|
||||
@@ -595,9 +558,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Enable batch-invariant mode: deterministic results regardless of
|
||||
# batch composition. Requires NVIDIA GPU with compute capability >= 9.0.
|
||||
"VLLM_BATCH_INVARIANT": lambda: bool(int(os.getenv("VLLM_BATCH_INVARIANT", "0"))),
|
||||
"VLLM_REPLICATE_EMBED": lambda: (
|
||||
os.getenv("VLLM_REPLICATE_EMBED", "0").strip().lower() in ("1", "true")
|
||||
),
|
||||
# Use tensor descriptors for Q/K/V loads and output stores in the
|
||||
# Triton unified-attention kernel. Enables HW 2D block reads on
|
||||
# Intel XPU; the non-TD branch is dead-code-eliminated at Triton
|
||||
@@ -640,10 +600,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
),
|
||||
# Build provenance metadata embedded in official vllm-openai images.
|
||||
# Set via Docker ENV at image build time; informational only.
|
||||
"VLLM_BUILD_COMMIT": lambda: os.environ.get("VLLM_BUILD_COMMIT", "unknown"),
|
||||
"VLLM_BUILD_PIPELINE": lambda: os.environ.get("VLLM_BUILD_PIPELINE", "local"),
|
||||
"VLLM_BUILD_URL": lambda: os.environ.get("VLLM_BUILD_URL", ""),
|
||||
"VLLM_IMAGE_TAG": lambda: os.environ.get("VLLM_IMAGE_TAG", ""),
|
||||
# CMake build type
|
||||
# If not set, defaults to "Debug" or "RelWithDebInfo"
|
||||
# Available options: "Debug", "Release", "RelWithDebInfo"
|
||||
@@ -839,13 +795,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
),
|
||||
# Pipeline stage partition strategy
|
||||
"VLLM_PP_LAYER_PARTITION": lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
|
||||
# (CPU backend only) CPU key-value cache space.
|
||||
# default is None and will be set as 4 GB
|
||||
"VLLM_CPU_KVCACHE_SPACE": lambda: (
|
||||
int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0"))
|
||||
if "VLLM_CPU_KVCACHE_SPACE" in os.environ
|
||||
else None
|
||||
),
|
||||
# (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31",
|
||||
# "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'.
|
||||
"VLLM_CPU_OMP_THREADS_BIND": lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "auto"),
|
||||
@@ -856,12 +805,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
if "VLLM_CPU_NUM_OF_RESERVED_CPU" in os.environ
|
||||
else None
|
||||
),
|
||||
# (CPU backend only) whether vLLM is running in a CI environment.
|
||||
"VLLM_CPU_CI_ENV": lambda: bool(int(os.getenv("VLLM_CPU_CI_ENV", "0"))),
|
||||
# (CPU backend only) whether to enable attention spilt KV.
|
||||
"VLLM_CPU_ATTN_SPLIT_KV": lambda: bool(
|
||||
int(os.getenv("VLLM_CPU_ATTN_SPLIT_KV", "1"))
|
||||
),
|
||||
# (Zen CPU backend) eagerly prepack weights into ZenDNN blocked layout
|
||||
# at model load time. Eliminates per-inference layout conversion overhead.
|
||||
"VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool(
|
||||
@@ -1025,16 +968,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"),
|
||||
# Path to the XLA persistent cache directory.
|
||||
# Only used for XLA devices such as TPUs.
|
||||
"VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser(
|
||||
os.getenv(
|
||||
"VLLM_XLA_CACHE_PATH",
|
||||
os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
|
||||
)
|
||||
),
|
||||
# If set, assert on XLA recompilation after each execution step.
|
||||
"VLLM_XLA_CHECK_RECOMPILATION": lambda: bool(
|
||||
int(os.getenv("VLLM_XLA_CHECK_RECOMPILATION", "0"))
|
||||
),
|
||||
# Enable SPMD mode for TPU backend.
|
||||
"VLLM_XLA_USE_SPMD": lambda: bool(int(os.getenv("VLLM_XLA_USE_SPMD", "0"))),
|
||||
# Maximum size (in MB) for logits tensor in sparse MLA indexer prefill chunks.
|
||||
@@ -1173,9 +1107,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
else os.environ["VLLM_DISABLED_KERNELS"].split(",")
|
||||
),
|
||||
# Selects hw-agnostic layers for HF transformer backend
|
||||
"VLLM_USE_HW_AGNOSTIC": lambda: (
|
||||
os.getenv("VLLM_USE_HW_AGNOSTIC", "False").lower() in ("true", "1")
|
||||
),
|
||||
"VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1"))
|
||||
),
|
||||
@@ -1277,12 +1208,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# context parallelism: "asm" uses AITER's round-robin ASM decode (gfx950),
|
||||
# "segmented" the Triton segmented MLA path. "auto" (default) takes "asm"
|
||||
# wherever it can serve the shape and "segmented" otherwise.
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices(
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY",
|
||||
"auto",
|
||||
["auto", "asm", "segmented"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# Small-head (<16) AITER MLA decode kernel selection. Small head counts
|
||||
# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
|
||||
# the Gluon small-head kernel or through the padded persistent-scheduling
|
||||
@@ -1291,12 +1216,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# PS ASM decode; "gluon" forces the Gluon path wherever a build exists;
|
||||
# "asm" forces the padded PS ASM decode. On gfx942/CDNA3 there is no Gluon
|
||||
# build, so the ASM path is always used regardless of this setting.
|
||||
"VLLM_ROCM_AITER_MLA_ASM_PADDING": env_with_choices(
|
||||
"VLLM_ROCM_AITER_MLA_ASM_PADDING",
|
||||
"auto",
|
||||
["auto", "gluon", "asm"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# Whether to use aiter mha ops.
|
||||
# By default is enabled.
|
||||
"VLLM_ROCM_USE_AITER_MHA": lambda: (
|
||||
@@ -1439,9 +1358,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Deprecated and ignored: RayExecutorV2 is the only Ray executor. Kept so
|
||||
# out-of-tree platforms that still read it keep working; will be removed
|
||||
# in v0.34.
|
||||
"VLLM_USE_RAY_V2_EXECUTOR_BACKEND": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_RAY_V2_EXECUTOR_BACKEND", "1"))
|
||||
),
|
||||
# In some system, find_loaded_library() may not work. So we allow users to
|
||||
# specify the path through environment variable VLLM_CUDART_SO_PATH.
|
||||
"VLLM_CUDART_SO_PATH": lambda: os.getenv("VLLM_CUDART_SO_PATH", None),
|
||||
@@ -1546,9 +1462,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
os.environ.get("VLLM_V1_USE_OUTLINES_CACHE", "0") == "1"
|
||||
),
|
||||
# Whether using Pathways
|
||||
"VLLM_TPU_USING_PATHWAYS": lambda: bool(
|
||||
"proxy" in os.getenv("JAX_PLATFORMS", "").lower()
|
||||
),
|
||||
# Allow use of DeepGemm kernels for fused moe ops.
|
||||
"VLLM_USE_DEEP_GEMM": lambda: bool(int(os.getenv("VLLM_USE_DEEP_GEMM", "1"))),
|
||||
# Allow use of DeepGemm specifically for MoE fused ops (overrides only MoE).
|
||||
@@ -1564,7 +1477,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
int(os.getenv("VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES", "1"))
|
||||
),
|
||||
# Opt-in MLA DCP query replication: skip the decode query all-gather.
|
||||
"VLLM_DCP_Q_REPLICATE": lambda: bool(int(os.getenv("VLLM_DCP_Q_REPLICATE", "0"))),
|
||||
# DeepGemm JITs the kernels on-demand. The warmup attempts to make DeepGemm
|
||||
# JIT all the required kernels before model execution so there is no
|
||||
# JIT'ing in the hot-path. However, this warmup increases the engine
|
||||
@@ -1602,24 +1514,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# per layer, so it only wins at low token counts: intended for decode
|
||||
# instances in a P/D disaggregated deployment, not for prefill or unified
|
||||
# serving.
|
||||
"VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool(
|
||||
int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0"))
|
||||
),
|
||||
# Kimi K3 only, and unrelated to the MoE flags above. Tap the pre-norm
|
||||
# AttnRes mixture, rather than the post-mixture sum, as the auxiliary
|
||||
# hidden state handed to a DFlash drafter. This changes the numerics the
|
||||
# speculator sees, so it is off by default while the effect is measured.
|
||||
"VLLM_KIMI_K3_AUX_ATTN_RES_STREAM": lambda: bool(
|
||||
int(os.getenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "0"))
|
||||
),
|
||||
# Use the SM100 BF16 GEMM-AR kernel for eligible Kimi-K3 row-parallel
|
||||
# attention projections. All TP ranks must belong to one NVLink domain.
|
||||
"VLLM_KIMI_K3_GEMM_AR": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_AR", "1"))),
|
||||
# Fuse eligible sequence-parallel row-parallel projections with their TP
|
||||
# reduce-scatter using the SM100 BF16/MXFP8 GEMM-RS kernel (Kimi-K3
|
||||
# attention/shared-expert projections, DeepSeek-V4.1 ``wo_b``). All TP
|
||||
# ranks must belong to one NVLink domain.
|
||||
"VLLM_ENABLE_GEMM_RS": lambda: bool(int(os.getenv("VLLM_ENABLE_GEMM_RS", "0"))),
|
||||
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
|
||||
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
|
||||
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
|
||||
@@ -1666,13 +1570,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
int(os.getenv("VLLM_DISABLE_REQUEST_ID_RANDOMIZATION", "0"))
|
||||
),
|
||||
# IP address used for NIXL handshake between remote agents.
|
||||
"VLLM_NIXL_SIDE_CHANNEL_HOST": lambda: os.getenv(
|
||||
"VLLM_NIXL_SIDE_CHANNEL_HOST", "localhost"
|
||||
),
|
||||
# Port used for NIXL handshake between remote agents.
|
||||
"VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(
|
||||
os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600")
|
||||
),
|
||||
# Address the P2P KV-offload control socket binds to. Defaults to
|
||||
# ``localhost`` (loopback only); must be set to the node IP for
|
||||
# cross-host P2P so remote peers can reach the socket.
|
||||
@@ -1700,39 +1598,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_P2P_IDLE_TIMEOUT_S": lambda: int(os.getenv("VLLM_P2P_IDLE_TIMEOUT_S", "300")),
|
||||
# IP address used for the EC connector's ZMQ side channel
|
||||
# (producer ROUTER bind, consumer DEALER dial).
|
||||
"VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv(
|
||||
"VLLM_EC_SIDE_CHANNEL_HOST", "localhost"
|
||||
),
|
||||
# Port for the EC connector's ZMQ side channel; advertised to peers
|
||||
# via `ec_transfer_params.peer_port` on the producer's response.
|
||||
"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
|
||||
os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
|
||||
),
|
||||
# Per-request timeout (seconds) when a prefiller worker registers with
|
||||
# the Mooncake bootstrap server.
|
||||
"VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float(
|
||||
os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0")
|
||||
),
|
||||
# Port used for Mooncake handshake between remote agents.
|
||||
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
|
||||
),
|
||||
# Log per-batch memory/disk tier breakdown on external GETs.
|
||||
"VLLM_MOONCAKE_STORE_TIER_LOG": lambda: (
|
||||
os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1")
|
||||
),
|
||||
# Number of parallel KV-load receive threads per worker rank. Lets the
|
||||
# per-request control overhead (Python prep + master key lookup) of one
|
||||
# request overlap with the RDMA transfer of another, keeping the transfer
|
||||
# engine's queue pairs busy. Helps when that overhead is significant or
|
||||
# per-request batches are too small to saturate the link on their own.
|
||||
"VLLM_MOONCAKE_LOAD_RECV_THREADS": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_LOAD_RECV_THREADS", "1")
|
||||
),
|
||||
# Fraction of the owner's DirectIO staging buffer to fill per GET batch.
|
||||
"VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float(
|
||||
os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9")
|
||||
),
|
||||
# Pin this rank to a specific owner segment ("host:port").
|
||||
"MOONCAKE_PREFERRED_SEGMENT": lambda: os.getenv("MOONCAKE_PREFERRED_SEGMENT"),
|
||||
# Override the hostname the rank registers as a Mooncake requester.
|
||||
@@ -1788,18 +1665,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MOE_ROUTING_SIMULATION_STRATEGY", ""
|
||||
).lower(),
|
||||
# Regex timeout for use by the vLLM tool parsing plugins.
|
||||
"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int(
|
||||
os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")
|
||||
),
|
||||
# Enforce function parameter schemas in structural-tag based tool calling.
|
||||
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: (
|
||||
os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Constrain every `beam` output to its response template's format, also
|
||||
# without tool or structured output constraints.
|
||||
"VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT": lambda: (
|
||||
os.getenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Control the max chunk bytes (in MB) for the rpc message queue.
|
||||
# Object larger than this threshold will be broadcast to worker
|
||||
# processes via zmq.
|
||||
@@ -1849,9 +1720,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
|
||||
),
|
||||
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
|
||||
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
|
||||
),
|
||||
# If set, it means we pre-downloaded cubin files and flashinfer will
|
||||
# read the cubin files directly.
|
||||
"VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
|
||||
@@ -1891,12 +1759,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
|
||||
),
|
||||
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``
|
||||
# and its render endpoint at ``POST /cohere/v2/chat/render``.
|
||||
# Default off
|
||||
"VLLM_ENABLE_COHERE_API": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
|
||||
),
|
||||
# If set, use the fp8 mfma in rocm paged attention.
|
||||
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
|
||||
int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
|
||||
@@ -2056,9 +1918,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Emergency rollback for the DeepSeek-V4 NVIDIA MegaMoE path. By default,
|
||||
# DeepGEMM computes replicated FP8 shared experts in the same persistent
|
||||
# SM100 kernel as the routed FP4 experts.
|
||||
"VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION": lambda: bool(
|
||||
int(os.getenv("VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION", "0"))
|
||||
),
|
||||
# Limits when we run shared_experts in a separate stream.
|
||||
# We found out that for large batch sizes, the separate stream
|
||||
# execution is not beneficial (most likely because of the input clone)
|
||||
@@ -2074,9 +1933,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# and the cross-stream sync becomes pure overhead. Set to 0 to disable
|
||||
# the multi-stream path entirely. See #PR 41526 for the empirical result
|
||||
# for the default value of 1024 tokens.
|
||||
"VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int(
|
||||
os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024")
|
||||
),
|
||||
# Format for saving torch.compile cache artifacts
|
||||
# - "binary": saves as binary file
|
||||
# Safe for multiple vllm serve processes accessing the same torch compile cache.
|
||||
@@ -2175,9 +2031,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# activations to per-token int8 (int4_gemm_w4a8) for large token counts.
|
||||
# The two oneDNN backends are int4-only; ARK also serves int2.
|
||||
# Which one is fastest is device-dependent, so this is left as an opt-in.
|
||||
"VLLM_XPU_INC_WNA16_BACKEND": env_with_choices(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND", "auto", ["auto", "ark", "w4a16", "w4a8"]
|
||||
),
|
||||
# Direct DCP ops default on when applicable; set to 1 to enforce or 0 to disable.
|
||||
"VLLM_USE_DIRECT_DCP_A2A": lambda: maybe_convert_bool(
|
||||
os.getenv("VLLM_USE_DIRECT_DCP_A2A")
|
||||
@@ -2212,7 +2065,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# fused into gated_mla_gemm; elementwise gating only.
|
||||
# Each op additionally checks its own shape / dtype constraints and falls
|
||||
# back to the eager path when they do not hold.
|
||||
"VLLM_ENABLE_HPC_OPS": lambda: bool(int(os.getenv("VLLM_ENABLE_HPC_OPS", "0"))),
|
||||
# Whether to skip version suffix when building package
|
||||
"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
|
||||
int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
|
||||
@@ -2308,13 +2160,11 @@ def compile_factors() -> dict[str, object]:
|
||||
# HOME or the XDG roots silently invalidates every compile cache
|
||||
# (VLLM_CACHE_ROOT above and VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR below
|
||||
# are already ignored for the same reason).
|
||||
"VLLM_XLA_CACHE_PATH",
|
||||
"VLLM_CONFIG_ROOT",
|
||||
"LD_LIBRARY_PATH",
|
||||
"VLLM_SERVER_DEV_MODE",
|
||||
"VLLM_DP_MASTER_IP",
|
||||
"VLLM_DP_MASTER_PORT",
|
||||
"VLLM_NIXL_SIDE_CHANNEL_HOST",
|
||||
"VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
|
||||
"VLLM_MODEL_REDIRECT_PATH",
|
||||
"VLLM_HOST_IP",
|
||||
@@ -2369,7 +2219,6 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_WORKER_MULTIPROC_METHOD",
|
||||
"VLLM_ENABLE_V1_MULTIPROCESSING",
|
||||
"VLLM_V1_OUTPUT_PROC_CHUNK_SIZE",
|
||||
"VLLM_CPU_KVCACHE_SPACE",
|
||||
"VLLM_CPU_MOE_PREPACK",
|
||||
"VLLM_ZENTORCH_WEIGHT_PREPACK",
|
||||
"VLLM_TEST_FORCE_LOAD_FORMAT",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,596 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utilities for Punica kernel construction."""
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _accumulate_mm(
|
||||
tiled_a,
|
||||
tiled_b,
|
||||
accumulator,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
iter_k,
|
||||
group_k: tl.constexpr,
|
||||
group_n: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
):
|
||||
"""Core matrix multiplication and accumulation logic with quantization support.
|
||||
|
||||
Args:
|
||||
tiled_a (tl.tensor): Loaded tile from A matrix
|
||||
tiled_b (tl.tensor): Loaded tile from B matrix
|
||||
accumulator (tl.tensor): Current accumulator value
|
||||
a_scale_ptr (tl.tensor): Scale pointer for A matrix
|
||||
b_scale_ptr (tl.tensor): Scale pointer for B matrix
|
||||
a_scale_k_stride (int): K dimension stride for A's block-wise scales
|
||||
b_scale_k_stride (int): K dimension stride for B's block-wise scales
|
||||
iter_k (int): Current iteration's global K offset
|
||||
group_k: Block size for K dimension in block-wise quantization
|
||||
group_n: Block size for N dimension in block-wise quantization
|
||||
use_fp8_w8a8: Whether using FP8 W8A8 quantization
|
||||
|
||||
"""
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise quantization: scales are loaded per block
|
||||
offs_ks = iter_k // group_k
|
||||
# a_scale_ptr is (BLOCK_M,) tensor of base pointers per row
|
||||
# Load scale for current K-group, result shape: (BLOCK_M,)
|
||||
a_scale = tl.load(a_scale_ptr + offs_ks * a_scale_k_stride)
|
||||
# b_scale_ptr is (BLOCK_N,) tensor with N-offset pre-baked
|
||||
# Load scale for current K-group, result shape: (BLOCK_N,)
|
||||
b_scale = tl.load(b_scale_ptr + offs_ks * b_scale_k_stride)
|
||||
accumulator += (
|
||||
tl.dot(tiled_a, tiled_b) * a_scale[:, None] * b_scale[None, :]
|
||||
)
|
||||
else:
|
||||
# Tensor-wise or per-channel: accumulate and scale at end
|
||||
accumulator = tl.dot(tiled_a, tiled_b, acc=accumulator)
|
||||
else:
|
||||
accumulator += tl.dot(tiled_a, tiled_b)
|
||||
return accumulator
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fp8_mm_k(
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
ak_stride,
|
||||
bk_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
offset_k,
|
||||
K: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
group_k: tl.constexpr,
|
||||
group_n: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
CAST_TYPE: tl.constexpr,
|
||||
b_dtype: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
base_k,
|
||||
):
|
||||
"""FP8-compatible matrix multiplication kernel with quantization support.
|
||||
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
|
||||
B (k x n), iterate through the K dimension to compute the partial/complete
|
||||
matrix block product with proper dequantization.
|
||||
|
||||
Args:
|
||||
a_ptr (tl.tensor): Array of pointers, identifying rows of A
|
||||
(FP8 or other dtype)
|
||||
b_ptr (tl.tensor): Array of pointers, identifying columns of B
|
||||
(FP8 dtype)
|
||||
a_scale_ptr (tl.tensor): Scale pointer for A matrix
|
||||
(per-token or block-wise)
|
||||
b_scale_ptr (tl.tensor): Scale pointer for B matrix
|
||||
(per-channel or block-wise)
|
||||
ak_stride (int): K dimension stride of the A matrix
|
||||
bk_stride (int): K dimension stride of the B matrix
|
||||
a_scale_k_stride (int): K dimension stride for A's block-wise scales
|
||||
b_scale_k_stride (int): K dimension stride for B's block-wise scales
|
||||
offset_k (int): Base offset along K dimension
|
||||
K: Length of the K dimension
|
||||
BLOCK_M: M dimension of the output block m x n
|
||||
BLOCK_N: N dimension of the output block m x n
|
||||
BLOCK_K: K dimension atom
|
||||
EVEN_K: True if the blocks of A and B can be loaded without masking
|
||||
SPLIT_K: Parameter signifying parallelism in the K dimension
|
||||
group_k: Block size for K dimension in block-wise quantization
|
||||
group_n: Block size for N dimension in block-wise quantization
|
||||
use_fp8_w8a8: Whether using FP8 W8A8 quantization
|
||||
per_channel_quant: Whether using per-channel quantization
|
||||
CAST_TYPE: if True, cast the values from the A matrix to the B
|
||||
matrix dtype.
|
||||
b_dtype: datatype of the B matrix
|
||||
USE_GDC: Whether to use PDL. True indicates use.
|
||||
base_k (int): Base offset along K dimension for current SPLIT_K group
|
||||
|
||||
"""
|
||||
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
|
||||
# Step size along K for each iteration
|
||||
STEP_K = BLOCK_K * SPLIT_K
|
||||
|
||||
# Total number of iterations (compile-time constant)
|
||||
num_iters = tl.cdiv(K, STEP_K)
|
||||
|
||||
for k in range(num_iters):
|
||||
# Current iteration's global K offset
|
||||
iter_k = k * STEP_K + base_k
|
||||
block_end = iter_k + BLOCK_K
|
||||
|
||||
# Skip iterations that are entirely past the K boundary
|
||||
if not EVEN_K and iter_k >= K:
|
||||
pass
|
||||
elif EVEN_K or block_end <= K:
|
||||
# No masking needed: either K is evenly divisible (EVEN_K)
|
||||
# or this block fits entirely within K
|
||||
tiled_b = tl.load(b_ptr)
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tiled_a = tl.load(a_ptr)
|
||||
if CAST_TYPE:
|
||||
tiled_a = tiled_a.to(b_dtype)
|
||||
|
||||
accumulator = _accumulate_mm(
|
||||
tiled_a,
|
||||
tiled_b,
|
||||
accumulator,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
iter_k,
|
||||
group_k,
|
||||
group_n,
|
||||
use_fp8_w8a8,
|
||||
)
|
||||
else:
|
||||
# Partial block at the tail: mask out-of-bounds elements
|
||||
k_offsets = tl.arange(0, BLOCK_K)
|
||||
mask = iter_k + k_offsets < K
|
||||
tiled_b = tl.load(b_ptr, mask=mask[:, None], other=0.0)
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tiled_a = tl.load(a_ptr, mask=mask[None, :], other=0.0)
|
||||
if CAST_TYPE:
|
||||
tiled_a = tiled_a.to(b_dtype)
|
||||
|
||||
accumulator = _accumulate_mm(
|
||||
tiled_a,
|
||||
tiled_b,
|
||||
accumulator,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
iter_k,
|
||||
group_k,
|
||||
group_n,
|
||||
use_fp8_w8a8,
|
||||
)
|
||||
|
||||
a_ptr += STEP_K * ak_stride
|
||||
b_ptr += STEP_K * bk_stride
|
||||
|
||||
return accumulator
|
||||
|
||||
|
||||
@triton.jit
|
||||
def do_shrink_kernel_fp8(
|
||||
pid_n,
|
||||
pid_sk,
|
||||
slice_id,
|
||||
lora_index,
|
||||
input_ptr,
|
||||
lora_ptr,
|
||||
out_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
N,
|
||||
K,
|
||||
M_LEN,
|
||||
ram,
|
||||
# input strides
|
||||
input_d0_stride,
|
||||
input_d1_stride,
|
||||
# lora strides
|
||||
lora_d0_stride,
|
||||
lora_d1_stride,
|
||||
lora_d2_stride,
|
||||
# scale strides
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
# output strides
|
||||
output_d0_stride,
|
||||
output_d1_stride,
|
||||
output_d2_stride,
|
||||
scaling,
|
||||
# block size for block-wise quantization
|
||||
group_n: tl.constexpr,
|
||||
group_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
SLICE_NUM: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice, compute the
|
||||
matrix product and store in the appropriate output location.
|
||||
"""
|
||||
# Identify the lora_ptr from slice_id.
|
||||
if SLICE_NUM == 1:
|
||||
cur_lora_ptr = lora_ptr
|
||||
cur_b_scale_ptr = b_scale_ptr
|
||||
else:
|
||||
cur_lora_ptr = (
|
||||
tl.load(lora_ptr + slice_id).to(tl.pointer_type(tl.float8e4nv))
|
||||
if b_scale_ptr is not None
|
||||
else tl.load(lora_ptr + slice_id).to(
|
||||
tl.pointer_type(input_ptr.dtype.element_ty)
|
||||
)
|
||||
)
|
||||
cur_b_scale_ptr = (
|
||||
tl.load(b_scale_ptr + slice_id).to(tl.pointer_type(tl.float32))
|
||||
if b_scale_ptr is not None
|
||||
else b_scale_ptr
|
||||
)
|
||||
|
||||
# Identify the column indices of B to process.
|
||||
offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
|
||||
rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
|
||||
|
||||
# Identify A and B block pointers
|
||||
offset_k = pid_sk * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
a_ptr = (
|
||||
input_ptr + ram[:, None] * input_d0_stride + offset_k[None, :] * input_d1_stride
|
||||
)
|
||||
b_ptr = (
|
||||
cur_lora_ptr
|
||||
+ lora_d0_stride * lora_index
|
||||
+ rbn[None, :] * lora_d1_stride
|
||||
+ offset_k[:, None] * lora_d2_stride
|
||||
)
|
||||
|
||||
# Load scales for tensor-wise or per-channel quantization (outside the loop)
|
||||
# Block-wise scales are loaded inside fp8_mm_k
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise: compute scale pointers for fp8_mm_k
|
||||
# a_scale: per-row base pointers, shape (BLOCK_M,)
|
||||
# Each pointer points to the start of that row's scale data
|
||||
mm_a_scale_ptr = a_scale_ptr + ram * a_scale_m_stride
|
||||
|
||||
# b_scale: pre-compute N-dimension offset
|
||||
# We need to bake in the N-group offset since fp8_mm_k doesn't know pid_n
|
||||
n_offset = pid_n * BLOCK_N
|
||||
offs_ns = (n_offset + tl.arange(0, BLOCK_N)) // group_n
|
||||
# Base pointer with lora offset + N-group offset baked in, shape (BLOCK_N,)
|
||||
mm_b_scale_ptr = (
|
||||
cur_b_scale_ptr
|
||||
+ lora_index * b_scale_l_stride
|
||||
+ offs_ns * b_scale_n_stride
|
||||
)
|
||||
elif per_channel_quant:
|
||||
# Per-channel for weights, per-token for activations
|
||||
b_scale_ptrs = (
|
||||
cur_b_scale_ptr + lora_index * b_scale_l_stride + rbn * b_scale_n_stride
|
||||
)
|
||||
b_scale = tl.load(b_scale_ptrs)
|
||||
# Per-token activation scale
|
||||
a_scale = tl.load(a_scale_ptr + ram * a_scale_m_stride)[:, None]
|
||||
# For non-block-wise, pass original pointers (not used in mm loop)
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
else:
|
||||
# Tensor-wise quantization
|
||||
a_scale = tl.load(a_scale_ptr) if a_scale_ptr is not None else 1.0
|
||||
b_scale = tl.load(cur_b_scale_ptr + lora_index * b_scale_l_stride)
|
||||
# For non-block-wise, pass original pointers (not used in mm loop)
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
else:
|
||||
# Non-quantized path
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
|
||||
# Compute partial/complete block matrix product.
|
||||
accumulator = fp8_mm_k(
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
mm_a_scale_ptr,
|
||||
mm_b_scale_ptr,
|
||||
input_d1_stride,
|
||||
lora_d2_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
offset_k,
|
||||
K,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
EVEN_K,
|
||||
SPLIT_K,
|
||||
group_k,
|
||||
group_n,
|
||||
use_fp8_w8a8,
|
||||
per_channel_quant,
|
||||
False,
|
||||
cur_lora_ptr.dtype.element_ty,
|
||||
USE_GDC,
|
||||
base_k=pid_sk * BLOCK_K,
|
||||
)
|
||||
# GDC launch dependents hints the runtime system to launch dependent kernels.
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
# Apply dequantization scales for tensor-wise/per-channel quantization
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise: already applied in fp8_mm_k
|
||||
pass
|
||||
else:
|
||||
# Tensor-wise or per-channel: apply scales after accumulation
|
||||
accumulator = accumulator * a_scale * b_scale
|
||||
|
||||
# Apply LoRA scaling factor
|
||||
accumulator *= scaling
|
||||
|
||||
# Identify the C output pointers to store the results of the accumulator.
|
||||
offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
|
||||
offset_cm = tl.arange(0, BLOCK_M)
|
||||
cur_out_ptr = out_ptr if SLICE_NUM == 1 else out_ptr + slice_id * output_d0_stride
|
||||
c_ptr = (
|
||||
cur_out_ptr
|
||||
+ ram[:, None] * output_d1_stride
|
||||
+ offset_cn[None, :] * output_d2_stride
|
||||
)
|
||||
c_mask = (offset_cm[:, None] < M_LEN) & (offset_cn[None, :] < N)
|
||||
|
||||
# Cast accumulator to output dtype
|
||||
accumulator = accumulator.to(out_ptr.dtype.element_ty)
|
||||
|
||||
# handles write-back with reduction-splitting
|
||||
if SPLIT_K == 1:
|
||||
tl.store(c_ptr, accumulator, mask=c_mask)
|
||||
else:
|
||||
tl.atomic_add(c_ptr, accumulator, mask=c_mask, sem="relaxed")
|
||||
|
||||
|
||||
@triton.jit
|
||||
def do_expand_kernel_fp8(
|
||||
pid_n,
|
||||
lora_index,
|
||||
slice_id,
|
||||
input_ptr,
|
||||
lora_ptr,
|
||||
out_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
N,
|
||||
K,
|
||||
M_LEN,
|
||||
ram, # array identifying the rows of Input ptr to operate on
|
||||
slice_start_loc,
|
||||
# input ptr strides
|
||||
input_d0_stride,
|
||||
input_d1_stride,
|
||||
input_d2_stride,
|
||||
# lora ptr strides
|
||||
ls_d0_ptr,
|
||||
ls_d1_ptr,
|
||||
ls_d2_ptr,
|
||||
# scale strides
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
# out ptr strides
|
||||
output_d0_stride,
|
||||
output_d1_stride,
|
||||
# block size for block-wise quantization
|
||||
group_n: tl.constexpr,
|
||||
group_k: tl.constexpr,
|
||||
# constants
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
SAME_STRIDE: tl.constexpr,
|
||||
SLICE_NUM: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
CAST_TYPE: tl.constexpr,
|
||||
ADD_INPUTS: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
):
|
||||
"""FP8-compatible expand kernel for LoRA.
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice,
|
||||
compute the matrix product with FP8 quantization support and store in
|
||||
the appropriate output location.
|
||||
|
||||
For expand kernel, the input (shrink output) may be in FP32/FP16/BF16,
|
||||
while the LoRA B weights can be in FP8.
|
||||
|
||||
Supports:
|
||||
- FP8 W8A8 quantization for LoRA B weights
|
||||
- Block-wise quantization with configurable group_k and group_n
|
||||
- Per-channel quantization
|
||||
- Tensor-wise quantization
|
||||
"""
|
||||
# ls_d*_ptr can be either an integer or a pointer
|
||||
if SAME_STRIDE:
|
||||
cur_lora_d0_stride = ls_d0_ptr
|
||||
cur_lora_d1_stride = ls_d1_ptr
|
||||
cur_lora_d2_stride = ls_d2_ptr
|
||||
else:
|
||||
cur_lora_d0_stride = tl.load(ls_d0_ptr + slice_id)
|
||||
cur_lora_d1_stride = tl.load(ls_d1_ptr + slice_id)
|
||||
cur_lora_d2_stride = tl.load(ls_d2_ptr + slice_id)
|
||||
|
||||
# Identify the input_ptr and lora_ptr from slice_id.
|
||||
if SLICE_NUM == 1:
|
||||
cur_input_ptr = input_ptr
|
||||
if use_fp8_w8a8:
|
||||
cur_lora_ptr = lora_ptr
|
||||
cur_b_scale_ptr = b_scale_ptr
|
||||
else:
|
||||
cur_lora_ptr = lora_ptr
|
||||
cur_b_scale_ptr = b_scale_ptr # May be None for non-quantized
|
||||
else:
|
||||
cur_input_ptr = input_ptr + slice_id * input_d0_stride
|
||||
if use_fp8_w8a8:
|
||||
cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
|
||||
tl.pointer_type(tl.float8e4nv)
|
||||
)
|
||||
cur_b_scale_ptr = tl.load(b_scale_ptr + slice_id).to(
|
||||
tl.pointer_type(tl.float32)
|
||||
)
|
||||
else:
|
||||
cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
|
||||
tl.pointer_type(out_ptr.dtype.element_ty)
|
||||
)
|
||||
cur_b_scale_ptr = (
|
||||
tl.load(b_scale_ptr + slice_id).to(tl.pointer_type(tl.float32))
|
||||
if b_scale_ptr is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Identify the column indices of B to process.
|
||||
offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
|
||||
rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
|
||||
|
||||
# Identify A and B block pointers
|
||||
offset_k = tl.arange(0, BLOCK_K)
|
||||
a_ptr = (
|
||||
cur_input_ptr
|
||||
+ ram[:, None] * input_d1_stride
|
||||
+ offset_k[None, :] * input_d2_stride
|
||||
)
|
||||
b_ptr = (
|
||||
cur_lora_ptr
|
||||
+ cur_lora_d0_stride * lora_index
|
||||
+ offset_k[:, None] * cur_lora_d2_stride
|
||||
+ rbn[None, :] * cur_lora_d1_stride
|
||||
)
|
||||
|
||||
# Setup scale pointers for FP8/INT8 quantization
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise quantization - compute scale pointers for fp8_mm_k
|
||||
# a_scale: per-row base pointers, shape (BLOCK_M,)
|
||||
mm_a_scale_ptr = a_scale_ptr + ram * a_scale_m_stride
|
||||
|
||||
# b_scale: pre-compute N-dimension offset since fp8_mm_k doesn't know pid_n
|
||||
n_offset = pid_n * BLOCK_N
|
||||
offs_ns = (n_offset + tl.arange(0, BLOCK_N)) // group_n
|
||||
# Base pointer with lora offset + N-group offset baked in, shape (BLOCK_N,)
|
||||
mm_b_scale_ptr = (
|
||||
cur_b_scale_ptr
|
||||
+ lora_index * b_scale_l_stride
|
||||
+ offs_ns * b_scale_n_stride
|
||||
)
|
||||
elif per_channel_quant:
|
||||
# Per-channel for weights, shape (BLOCK_N,)
|
||||
b_scale_ptrs = (
|
||||
cur_b_scale_ptr + lora_index * b_scale_l_stride + rbn * b_scale_n_stride
|
||||
)
|
||||
b_scale = tl.load(b_scale_ptrs)
|
||||
# Per-token activation scale, only if a_scale_ptr provided
|
||||
a_scale = tl.load(a_scale_ptr + ram * a_scale_m_stride)[:, None]
|
||||
# For non-block-wise, pass original pointers (not used in mm loop)
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
else:
|
||||
# Tensor-wise quantization
|
||||
a_scale = tl.load(a_scale_ptr) if a_scale_ptr is not None else 1.0
|
||||
b_scale = tl.load(cur_b_scale_ptr + lora_index * b_scale_l_stride)
|
||||
# For non-block-wise, pass original pointers (not used in mm loop)
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
else:
|
||||
# Non-quantized path
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
|
||||
# Compute the block matrix product using fp8_mm_k
|
||||
# Note: For expand kernel, SPLIT_K=1, so we pass 1 for SPLIT_K
|
||||
accumulator = fp8_mm_k(
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
mm_a_scale_ptr,
|
||||
mm_b_scale_ptr,
|
||||
input_d2_stride, # ak_stride
|
||||
cur_lora_d2_stride, # bk_stride
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
offset_k,
|
||||
K,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
EVEN_K,
|
||||
1, # SPLIT_K = 1 for expand kernel
|
||||
group_k,
|
||||
group_n,
|
||||
use_fp8_w8a8,
|
||||
per_channel_quant,
|
||||
CAST_TYPE, # CAST_TYPE - cast FP8 B to A's dtype
|
||||
cur_lora_ptr.dtype.element_ty,
|
||||
USE_GDC,
|
||||
base_k=0,
|
||||
)
|
||||
|
||||
# Apply dequantization scales for non-block-wise quantization
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
pass # Already applied per block in fp8_mm_k
|
||||
else:
|
||||
# Tensor-wise or per-channel: apply scales after accumulation
|
||||
accumulator = accumulator * a_scale * b_scale
|
||||
|
||||
tiled_c = accumulator.to(out_ptr.dtype.element_ty)
|
||||
if SLICE_NUM == 1:
|
||||
cur_slice_start = slice_start_loc
|
||||
else:
|
||||
cur_slice_start = tl.load(slice_start_loc + slice_id)
|
||||
|
||||
# Identify the C output pointers to store the results of the accumulator.
|
||||
offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + cur_slice_start
|
||||
offset_cm = tl.arange(0, BLOCK_M)
|
||||
c_ptr = (
|
||||
out_ptr
|
||||
+ ram[:, None] * output_d0_stride
|
||||
+ offset_cn[None, :] * output_d1_stride
|
||||
)
|
||||
c_mask = (offset_cm[:, None] < M_LEN) & (offset_cn[None, :] < (cur_slice_start + N))
|
||||
|
||||
if ADD_INPUTS:
|
||||
tiled_out = tl.load(c_ptr, mask=c_mask)
|
||||
tiled_c += tiled_out
|
||||
tl.store(c_ptr, tiled_c, mask=c_mask)
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 2
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
}
|
||||
}
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"9216": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"13312": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"17408": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"25600": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"33792": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"41984": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"50176": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"58368": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"9216": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"13312": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"17408": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"25600": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"33792": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"41984": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"50176": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"58368": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"9216": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"13312": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"17408": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"25600": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"33792": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"41984": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"50176": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"58368": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
}
|
||||
}
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4
|
||||
},
|
||||
"9216": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"13312": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"17408": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"25600": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4
|
||||
},
|
||||
"33792": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"41984": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"50176": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"58368": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 256,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"9216": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"13312": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"17408": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"25600": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"33792": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"41984": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"50176": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"58368": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"9216": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"13312": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"17408": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"25600": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"33792": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"41984": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"50176": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"58368": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"9216": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"13312": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"17408": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"25600": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"33792": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"41984": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"50176": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"58368": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 5
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 64,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"9216": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"13312": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"17408": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"25600": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"33792": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"41984": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"50176": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"58368": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 256,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user