Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 18:09:11 +00:00
43 changed files with 1290 additions and 380 deletions
@@ -27,6 +27,7 @@ from .utils import (
BeamSearchOutput,
BeamSearchSequence,
create_sort_beams_key_function,
get_beam_search_eos_token_ids,
)
logger = init_logger(__name__)
@@ -86,8 +87,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
self.llm_engine.input_processor.resolve_watermarking(params)
tokenizer = self.renderer.get_tokenizer()
eos_token_id = tokenizer.eos_token_id
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
engine_inputs = self._preprocess_cmpl(prompts)
lora_requests = self._lora_request_to_seq(lora_request, len(engine_inputs))
@@ -129,6 +128,11 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
params.structured_outputs, tokenizer
)
eos_token_ids = get_beam_search_eos_token_ids(
self.llm_engine.input_processor.generation_config_fields,
tokenizer.eos_token_id,
)
sort_beams_key = create_sort_beams_key_function(eos_token_ids, length_penalty)
instances: list[BeamSearchInstance] = []
for lora_req, prompt in zip(lora_requests, engine_inputs):
@@ -169,7 +173,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
should_stop = self._beam_search_step(
instances_batch=instances_batch,
base_sampling_params=base_sampling_params,
eos_token_id=eos_token_id,
eos_token_ids=eos_token_ids,
ignore_eos=ignore_eos,
beam_width=beam_width,
sort_beams_key=sort_beams_key,
@@ -204,7 +208,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
self,
instances_batch: list[BeamSearchInstance],
base_sampling_params: SamplingParams,
eos_token_id: int | None,
eos_token_ids: set[int],
ignore_eos: bool,
beam_width: int,
sort_beams_key: Callable,
@@ -350,7 +354,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
cum_logprob=current_beam.cum_logprob + logprob_obj.logprob,
)
if token_id == eos_token_id and not ignore_eos:
if token_id in eos_token_ids and not ignore_eos:
instance.completed.append(new_beam)
else:
instance_new_beams.append(new_beam)
@@ -20,6 +20,7 @@ from vllm.utils.async_utils import collect_from_async_generator
from .utils import (
BeamSearchSequence,
create_sort_beams_key_function,
get_beam_search_eos_token_ids,
)
@@ -47,8 +48,6 @@ class BeamSearchOnlineMixin(ABC):
self.engine_client.input_processor.resolve_watermarking(params)
tokenizer = self.renderer.get_tokenizer()
eos_token_id = tokenizer.eos_token_id
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
if prompt["type"] == "embeds":
raise NotImplementedError("Embedding prompt not supported for beam search")
@@ -74,6 +73,11 @@ class BeamSearchOnlineMixin(ABC):
sampling_params, self.engine_client.input_processor.vllm_config
)
allowed = set(params.allowed_token_ids) if params.allowed_token_ids else None
eos_token_ids = get_beam_search_eos_token_ids(
self.engine_client.input_processor.generation_config_fields,
tokenizer.eos_token_id,
)
sort_beams_key = create_sort_beams_key_function(eos_token_ids, length_penalty)
all_beams = [
BeamSearchSequence(
orig_prompt=prompt,
@@ -159,17 +163,17 @@ class BeamSearchOnlineMixin(ABC):
candidate_logprob = (
current_beam.cum_logprob + logprob_obj.logprob
)
if token_id == eos_token_id and not ignore_eos:
if token_id in eos_token_ids and not ignore_eos:
completed.append(
BeamSearchSequence(
orig_prompt=prompt,
tokens=current_beam.tokens + [eos_token_id]
tokens=current_beam.tokens + [token_id]
if include_stop_str_in_output
else current_beam.tokens,
logprobs=current_beam.logprobs + [logprobs],
cum_logprob=candidate_logprob,
finish_reason="stop",
stop_reason=eos_token_id,
stop_reason=token_id,
)
)
else:
@@ -219,7 +223,7 @@ class BeamSearchOnlineMixin(ABC):
best_beams = sorted_completed[:beam_width]
for beam in best_beams:
if beam.tokens[-1] == eos_token_id and not ignore_eos:
if beam.tokens[-1] in eos_token_ids and not ignore_eos:
# Skip the eos token in the text.
tokens = beam.tokens[tokenized_length:-1]
else:
+20 -4
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
from vllm.inputs import (
DecoderOnlyEngineInput,
@@ -13,6 +14,7 @@ from vllm.inputs import (
)
from vllm.logprobs import Logprob
from vllm.lora.request import LoRARequest
from vllm.sampling_params import SamplingParams
@dataclass
@@ -137,7 +139,7 @@ class BeamSearchInstance:
def get_beam_search_score(
tokens: list[int],
cumulative_logprob: float,
eos_token_id: int,
eos_token_ids: set[int],
length_penalty: float = 1.0,
) -> float:
"""Calculate the beam search score with length penalty.
@@ -147,17 +149,31 @@ def get_beam_search_score(
https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
"""
seq_len = len(tokens)
if tokens[-1] == eos_token_id:
if tokens[-1] in eos_token_ids:
seq_len -= 1
# An aborted beam may contain only an EOS prompt token.
return cumulative_logprob / (max(seq_len, 1) ** length_penalty)
def create_sort_beams_key_function(eos_token_id: int, length_penalty: float):
def create_sort_beams_key_function(eos_token_ids: set[int], length_penalty: float):
def sort_beams_key(x: BeamSearchSequence) -> float:
return get_beam_search_score(
x.tokens, x.cum_logprob, eos_token_id, length_penalty
x.tokens, x.cum_logprob, eos_token_ids, length_penalty
)
return sort_beams_key
def get_beam_search_eos_token_ids(
generation_config: dict[str, Any], eos_token_id: int | None
) -> set[int]:
"""Return the token ids that end generation: the tokenizer EOS plus the
generation config's ``eos_token_id``.
This uses a separate SamplingParams so the params sent to the engine keep
their own ``stop_token_ids`` and are validated as before.
"""
params = SamplingParams()
params.update_from_generation_config(generation_config, eos_token_id)
return params.all_stop_token_ids
@@ -9,14 +9,17 @@ import torch.nn.functional as F
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
from vllm.exceptions import ChatTemplateResolutionError
from vllm.logger import init_logger
from vllm.renderers import TokenizeParams
from vllm.renderers.hf import safe_apply_chat_template
from vllm.renderers import TokenizeParams, merge_kwargs
from vllm.renderers.hf import resolve_chat_template, safe_apply_chat_template
from vllm.renderers.inputs.preprocess import (
extract_target_prompt,
parse_model_prompt,
prompt_to_seq,
)
from vllm.tasks import PoolingTask
from vllm.transformers_utils.config import (
get_sentence_transformers_chat_template_kwargs,
)
from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
@@ -457,9 +460,22 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
self.model = model if self.supports_score_template else None
self.use_sep_token = self.model_config.use_sep_token
self.saved_chat_template: str | None = None
self.saved_chat_template_kwargs: dict[str, Any] = {}
if not self.supports_score_template:
try:
self.saved_chat_template, self.saved_chat_template_kwargs = (
self._get_saved_chat_template()
)
except ValueError:
# An explicit chat template takes precedence anyway.
if self.chat_template is None:
raise
if (
getattr(self.model_config.hf_config, "is_original_qwen3_reranker", False)
and self.chat_template is None
and self.saved_chat_template is None
):
suggested_template = (
"examples/pooling/score/template/qwen3_vl_reranker.jinja"
@@ -475,6 +491,41 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
suggested_template,
)
def _get_saved_chat_template(self) -> tuple[str | None, dict[str, Any]]:
"""Get the chat template and kwargs declared by the Sentence
Transformers config, if any."""
saved_kwargs = get_sentence_transformers_chat_template_kwargs(
self.model_config.model, self.model_config.revision
)
if saved_kwargs is None:
return None, {}
saved_kwargs = dict(saved_kwargs)
name = saved_kwargs.pop("chat_template", None)
if name == "default":
name = None
template = resolve_chat_template(
self.tokenizer,
chat_template=name,
tools=None,
model_config=self.model_config,
)
# Hugging Face returns an unknown template name unchanged.
if template is None or template == name:
raise ValueError(
"The Sentence Transformers config declares message inputs, but "
f"the checkpoint has no {name or 'default'!r} chat template."
)
if self.chat_template is None:
logger.info_once(
"Scoring with the %r chat template declared by the Sentence "
"Transformers config. Its suffix is not restored after "
"truncation; use max_tokens_per_doc to truncate documents.",
name or "default",
)
return template, saved_kwargs
#######################################
# online APIs
@@ -648,6 +699,12 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
model_config = self.model_config
tokenizer = self.tokenizer
if chat_template is None and self.saved_chat_template is not None:
chat_template = self.saved_chat_template
chat_template_kwargs = merge_kwargs(
self.saved_chat_template_kwargs, chat_template_kwargs
)
prompt_1, prompt_2, mm_data, mm_uuids = parse_score_data(
data_1,
data_2,
@@ -703,7 +760,8 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
prompt_inputs = tokenizer(text=full_prompt, **local_kwargs)
return full_prompt, prompt_inputs
# FIXME: For now, we only apply a template when one is explicitly provided.
# FIXME: For now, we only apply a template when one is explicitly provided,
# either directly or by the Sentence Transformers config.
# We cannot rely on the tokenizer's chat template because many models
# inherit junk templates from their base LLM, which breaks both the models
# and the tests that use them.
+39 -6
View File
@@ -82,6 +82,8 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950, on_gfx1250
from vllm.sequence import IntermediateTensors
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
from vllm.utils.torch_utils import aux_stream
from vllm.v1.worker.ubatching import dbo_current_ubatch_token_offset
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
@@ -713,6 +715,18 @@ class DeepseekV4MoE(nn.Module):
),
prefix=f"{prefix}.experts",
)
self.shared_experts_stream = (
aux_stream()
if self.shared_experts is not None
and not self.experts.fuse_shared_expert
and not envs.VLLM_DISABLE_SHARED_EXPERTS_STREAM
else None
)
self.shared_experts_events = (
(torch.cuda.Event(), torch.cuda.Event())
if self.shared_experts_stream is not None
else None
)
return
fuse_shared_into_routed = (
@@ -777,6 +791,30 @@ class DeepseekV4MoE(nn.Module):
def _forward_mega_moe(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None
) -> torch.Tensor:
shared_experts = self.shared_experts
if shared_experts is None:
return self._forward_mega_moe_routed(hidden_states, input_ids)
events = self.shared_experts_events
if (
events is None
or hidden_states.shape[0] > envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
):
return self._forward_mega_moe_routed(
hidden_states, input_ids
) + shared_experts(hidden_states)
# The unfused shared expert overlaps router + MegaMoE in small steps.
routed, shared = maybe_execute_in_parallel(
fn0=lambda: self._forward_mega_moe_routed(hidden_states, input_ids),
fn1=lambda: shared_experts(hidden_states),
event0=events[0],
event1=events[1],
aux_stream=self.shared_experts_stream,
)
return routed + shared
def _forward_mega_moe_routed(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None
) -> torch.Tensor:
router_logits, _ = self.gate(hidden_states)
topk_weights, topk_ids = fused_topk_bias(
@@ -792,12 +830,7 @@ class DeepseekV4MoE(nn.Module):
bias_vl=self.gate.bias_vl,
image_sentinel_lo=self.image_sentinel_lo,
)
final_hidden_states = self.experts(hidden_states, topk_weights, topk_ids)
if self.shared_experts is not None:
final_hidden_states = final_hidden_states + self.shared_experts(
hidden_states
)
return final_hidden_states
return self.experts(hidden_states, topk_weights, topk_ids)
# Hidden sizes supported by AITER mhc_pre_big_fuse_rmsnorm.
+15 -15
View File
@@ -70,7 +70,7 @@ class GatedResidual(nn.Module):
) -> None:
super().__init__()
self.config = config
self.lora_rank = config.hc_lowrank
self.hc_lowrank = config.hc_lowrank
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
self.use_combine = use_combine
@@ -91,11 +91,11 @@ class GatedResidual(nn.Module):
# -- vLLM Linear weights --------------------------------------------
# The merged skinny-GEMM shape is physically padded to 16 rows for
# alignment and efficient backend dispatch.
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
self.pad_size = (-(self.hc_lowrank + self.hc_count)) % 16 if use_combine else 0
if use_combine:
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
self.hyper_hidden_size,
[self.lora_rank, self.hc_count]
[self.hc_lowrank, self.hc_count]
+ ([self.pad_size] if self.pad_size else []),
bias=False,
params_dtype=config.params_dtype,
@@ -107,7 +107,7 @@ class GatedResidual(nn.Module):
else:
self.input_mix_weight_down = ReplicatedLinear(
self.hyper_hidden_size,
self.lora_rank,
self.hc_lowrank,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
@@ -115,7 +115,7 @@ class GatedResidual(nn.Module):
return_bias=False,
)
self.input_mix_weight_up = ReplicatedLinear(
self.lora_rank,
self.hc_lowrank,
self.hyper_hidden_size,
bias=False,
params_dtype=config.params_dtype,
@@ -136,15 +136,15 @@ class GatedResidual(nn.Module):
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
split_sizes = [self.hc_lowrank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
lowrank, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
lowrank = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
lowrank = hc_silu(lowrank, self.hc_count)
gate = self.input_mix_weight_up(lowrank) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
@@ -172,15 +172,15 @@ class GatedResidual(nn.Module):
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
split_sizes = [self.hc_lowrank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
lowrank, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
lowrank = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
lowrank = hc_silu(lowrank, self.hc_count)
gate = self.input_mix_weight_up(lowrank) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
+28 -3
View File
@@ -29,6 +29,7 @@ from vllm.models.common.ops.sequence_parallel import (
sp_reduce_scatter,
sp_shard,
)
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import (
@@ -242,7 +243,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
persistent=True,
)
divisor = int(config.make_ngram_vocab_size_divisible_by)
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
padded_vocab_size = round_up(total_vocab_size, divisor)
if params_dtype is None:
params_dtype = torch.get_default_dtype()
embedding_prefix = f"{prefix}.ngram_embedding"
@@ -280,9 +281,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
reduce_results=not self.use_reduce_scatter,
)
logger.info(
"Initialized AMD PLE embedding %s: quantization_method=%s, "
"Initialized AMD PLE embedding %s: "
"quantization_config=%s, quantization_method=%s, "
"weight_dtype=%s, weight_device=%s, pinned=%s",
embedding_prefix,
type(quant_config).__name__,
type(embedding_quant_method).__name__,
self.ngram_embedding.weight.dtype,
self.ngram_embedding.weight.device,
@@ -742,7 +745,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
)
kv, _ = self.kv_proj(embeddings)
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
gated_output, conv_input = ple_gate(
gated_output, conv_input = torch.ops.vllm.qwen4_exp_ple_gate(
key,
value,
hidden_states,
@@ -848,6 +851,28 @@ def qwen4_exp_ple_short_conv(
layer._short_conv(inputs, residual, outer_residual)
def _ple_gate_fake(
key: torch.Tensor,
value: torch.Tensor,
hidden: torch.Tensor,
norm_key_w: torch.Tensor,
norm_query_w: torch.Tensor,
norm_conv_w: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return the gate output shapes for AMD graph tracing."""
return torch.empty_like(hidden), torch.empty_like(hidden)
# Keep the AMD gate opaque to Dynamo while reusing the shared Triton kernel.
direct_register_custom_op(
op_name="qwen4_exp_ple_gate",
op_func=ple_gate,
mutates_args=[],
fake_impl=_ple_gate_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_amd_ple_ngram_ids",
op_func=qwen4_exp_amd_ple_ngram_ids,
+39 -36
View File
@@ -194,22 +194,21 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
)
if quant_config is None:
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
quant_algo = quant_config._resolve_quant_algo(prefix)
if quant_algo == "FP8":
return Qwen4ExpPLEFp8EmbeddingMethod()
if quant_algo == "NVFP4":
return Qwen4ExpPLENvFp4EmbeddingMethod()
if quant_algo is not None:
raise ValueError(
f"Unsupported quant_algo {quant_algo!r} for the PLE embedding "
f"{prefix!r}; expected FP8 or NVFP4"
)
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if isinstance(
quant_config, ModelOptQuantConfigBase
) and quant_config.is_layer_excluded(prefix):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if isinstance(quant_config, ModelOptQuantConfigBase):
if quant_config.is_layer_excluded(prefix):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
quant_algo = quant_config._resolve_quant_algo(prefix)
if quant_algo == "FP8":
return Qwen4ExpPLEFp8EmbeddingMethod()
if quant_algo == "NVFP4":
return Qwen4ExpPLENvFp4EmbeddingMethod()
if quant_algo is not None:
raise ValueError(
f"Unsupported quant_algo {quant_algo!r} for the PLE embedding "
f"{prefix!r}; expected FP8 or NVFP4"
)
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if (
isinstance(quant_config, CompressedTensorsConfig)
and quant_config.get_scheme_dict(None, layer_name=prefix) is None
@@ -223,29 +222,32 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
# Quark quantizes only Linear and MoE layers; PLE tables stay BF16.
if isinstance(quant_config, QuarkConfig):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if not isinstance(quant_config, Fp8Config):
raise NotImplementedError(
"Qwen4Exp PLE embedding does not support quantization config "
f"{type(quant_config).__name__}"
)
ignored_layers = quant_config.ignored_layers
if is_layer_skipped(
prefix,
ignored_layers,
quant_config.packed_modules_mapping,
match_mode=quant_config.ignored_layers_match_mode,
):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
# PLE checkpoint shards form one runtime embedding parameter.
shard_prefix = f"{prefix}.shard_"
if any(name.startswith(shard_prefix) for name in ignored_layers):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if not quant_config.is_checkpoint_fp8_serialized:
if isinstance(quant_config, Fp8Config):
ignored_layers = quant_config.ignored_layers
if is_layer_skipped(
prefix,
ignored_layers,
quant_config.packed_modules_mapping,
match_mode=quant_config.ignored_layers_match_mode,
):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
# PLE checkpoint shards form one runtime embedding parameter.
shard_prefix = f"{prefix}.shard_"
if any(name.startswith(shard_prefix) for name in ignored_layers):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if quant_config.is_checkpoint_fp8_serialized:
return Qwen4ExpPLEFp8EmbeddingMethod()
raise NotImplementedError(
"Qwen4Exp PLE embedding only supports serialized FP8 checkpoints"
)
return Qwen4ExpPLEFp8EmbeddingMethod()
logger.warning(
"Falling back to unquantized PLE embedding for %s "
"with quantization config %s.",
prefix,
type(quant_config).__name__,
)
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
def apply(
self,
@@ -781,6 +783,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
)
active_output = buffer[: gathered_ids.shape[0]]
prefetch_stream.wait_stream(torch.cuda.current_stream())
# Prevent ID storage reuse until the prefetch stream finishes reading it.
gathered_ids.record_stream(prefetch_stream)
with torch.cuda.stream(prefetch_stream):
self._lookup(gathered_ids, output=active_output)
+1 -36
View File
@@ -16,7 +16,6 @@ import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
@@ -240,7 +239,7 @@ def _ple_gate_kernel(
tl.store(normed_ptr + t * HC * H + offs, normed, mask=mask)
def _ple_gate(
def ple_gate(
key: torch.Tensor,
value: torch.Tensor,
hidden: torch.Tensor,
@@ -281,40 +280,6 @@ def _ple_gate(
return gated, normed
def _ple_gate_fake(
key: torch.Tensor,
value: torch.Tensor,
hidden: torch.Tensor,
norm_key_w: torch.Tensor,
norm_query_w: torch.Tensor,
norm_conv_w: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.empty_like(hidden), torch.empty_like(hidden)
direct_register_custom_op(
op_name="qwen4_exp_ple_gate",
op_func=_ple_gate,
mutates_args=[],
fake_impl=_ple_gate_fake,
)
def ple_gate(
key: torch.Tensor,
value: torch.Tensor,
hidden: torch.Tensor,
norm_key_w: torch.Tensor,
norm_query_w: torch.Tensor,
norm_conv_w: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.qwen4_exp_ple_gate(
key, value, hidden, norm_key_w, norm_query_w, norm_conv_w, eps
)
# Dilated short convolution
@triton.jit(do_not_specialize=["num_reqs", "bs_iters", "has_token_map"])
def _ple_conv_kernel(
+56 -60
View File
@@ -20,66 +20,64 @@ class PLEShardOverlap:
row_count: int
def compute_ple_shard_overlap(
*,
checkpoint_start: int,
checkpoint_rows: int,
tp_start: int,
tp_end: int,
) -> PLEShardOverlap | None:
"""Compute the overlap of a checkpoint shard and one TP vocabulary range."""
if checkpoint_start < 0 or checkpoint_rows < 0:
raise ValueError("checkpoint shard bounds must be non-negative")
if tp_start < 0 or tp_end < tp_start:
raise ValueError("invalid TP vocabulary range")
checkpoint_end = checkpoint_start + checkpoint_rows
overlap_start = max(checkpoint_start, tp_start)
overlap_end = min(checkpoint_end, tp_end)
if overlap_start >= overlap_end:
return None
return PLEShardOverlap(
source_start=overlap_start - checkpoint_start,
destination_start=overlap_start - tp_start,
row_count=overlap_end - overlap_start,
)
def copy_ple_embedding_shard_(
destination: torch.Tensor,
loaded_weight: torch.Tensor,
*,
checkpoint_start: int,
tp_start: int,
tp_end: int,
) -> int:
"""Copy the overlapping rows of a PLE checkpoint shard into a TP table."""
if destination.ndim == 0 or loaded_weight.ndim != destination.ndim:
raise ValueError("destination and loaded weight must have matching ranks")
if destination.shape[1:] != loaded_weight.shape[1:]:
raise ValueError(
"embedding shard dimensions do not match: "
f"{tuple(destination.shape[1:])} != {tuple(loaded_weight.shape[1:])}"
)
if destination.shape[0] < tp_end - tp_start:
raise ValueError("destination does not cover the requested TP range")
overlap = compute_ple_shard_overlap(
checkpoint_start=checkpoint_start,
checkpoint_rows=loaded_weight.shape[0],
tp_start=tp_start,
tp_end=tp_end,
)
if overlap is None:
return 0
source = loaded_weight.narrow(0, overlap.source_start, overlap.row_count)
target = destination.narrow(0, overlap.destination_start, overlap.row_count)
with torch.no_grad():
target.copy_(source.to(device=target.device, dtype=target.dtype))
return overlap.row_count
class PLEVocabParallelEmbedding(VocabParallelEmbedding):
"""Vocab-parallel embedding that accepts checkpoint row shards."""
def _compute_shard_overlap(
self,
*,
checkpoint_start: int,
checkpoint_rows: int,
) -> PLEShardOverlap | None:
"""Compute the overlap of a checkpoint shard and one TP vocabulary range."""
tp_start = self.shard_indices.org_vocab_start_index
tp_end = self.shard_indices.org_vocab_end_index
if checkpoint_start < 0 or checkpoint_rows < 0:
raise ValueError("checkpoint shard bounds must be non-negative")
if tp_start < 0 or tp_end < tp_start:
raise ValueError("invalid TP vocabulary range")
checkpoint_end = checkpoint_start + checkpoint_rows
overlap_start = max(checkpoint_start, tp_start)
overlap_end = min(checkpoint_end, tp_end)
if overlap_start >= overlap_end:
return None
return PLEShardOverlap(
source_start=overlap_start - checkpoint_start,
destination_start=overlap_start - tp_start,
row_count=overlap_end - overlap_start,
)
def _copy_embedding_shard_(
self,
destination: torch.Tensor,
loaded_weight: torch.Tensor,
*,
checkpoint_start: int,
) -> int:
"""Copy the overlapping rows of a PLE checkpoint shard into a TP table."""
tp_start = self.shard_indices.org_vocab_start_index
tp_end = self.shard_indices.org_vocab_end_index
if destination.ndim == 0 or loaded_weight.ndim != destination.ndim:
raise ValueError("destination and loaded weight must have matching ranks")
if destination.shape[1:] != loaded_weight.shape[1:]:
raise ValueError(
"embedding shard dimensions do not match: "
f"{tuple(destination.shape[1:])} != {tuple(loaded_weight.shape[1:])}"
)
if destination.shape[0] < tp_end - tp_start:
raise ValueError("destination does not cover the requested TP range")
overlap = self._compute_shard_overlap(
checkpoint_start=checkpoint_start,
checkpoint_rows=loaded_weight.shape[0],
)
if overlap is None:
return 0
source = loaded_weight.narrow(0, overlap.source_start, overlap.row_count)
target = destination.narrow(0, overlap.destination_start, overlap.row_count)
with torch.no_grad():
target.copy_(source.to(device=target.device, dtype=target.dtype))
return overlap.row_count
def weight_loader(
self,
param: torch.Tensor,
@@ -89,10 +87,8 @@ class PLEVocabParallelEmbedding(VocabParallelEmbedding):
if checkpoint_start is None:
super().weight_loader(param, loaded_weight)
return
copy_ple_embedding_shard_(
self._copy_embedding_shard_(
param,
loaded_weight,
checkpoint_start=checkpoint_start,
tp_start=self.shard_indices.org_vocab_start_index,
tp_end=self.shard_indices.org_vocab_end_index,
)
+3 -5
View File
@@ -24,7 +24,7 @@ from vllm.config.cache import CacheDType
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.math_utils import cdiv, round_up
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -880,9 +880,7 @@ class QSAKeyStateCache(_QSAStateCache):
key_head_size = int(kwargs.pop("head_size"))
self.key_head_size = key_head_size
self.cache_rope_positions = bool(cache_rope_positions)
self.rope_position_offset = (
(key_head_size + self._BF16_PER_INT64 - 1) // self._BF16_PER_INT64
) * self._BF16_PER_INT64
self.rope_position_offset = round_up(key_head_size, self._BF16_PER_INT64)
storage_head_size = key_head_size
if self.cache_rope_positions:
storage_head_size = self.rope_position_offset + (
@@ -910,7 +908,7 @@ class QSAKeyStateCache(_QSAStateCache):
# the scheduler block size). Anything narrower lets a rejected draft row
# overwrite a committed key the next step needs to close the group.
span = self.compress_ratio + vllm_config.num_speculative_tokens
capacity = self.compress_ratio * cdiv(span, self.compress_ratio)
capacity = round_up(span, self.compress_ratio)
assert self.cache_config.block_size % capacity == 0, (
f"QSA ring capacity {capacity} must divide the attention block "
f"size {self.cache_config.block_size}"
+13 -13
View File
@@ -74,7 +74,7 @@ class GatedResidual(nn.Module):
) -> None:
super().__init__()
self.config = config
self.lora_rank = config.hc_lowrank
self.hc_lowrank = config.hc_lowrank
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
self.use_combine = use_combine
@@ -95,11 +95,11 @@ class GatedResidual(nn.Module):
# -- vLLM Linear weights --------------------------------------------
# The merged skinny-GEMM shape is physically padded to 16 rows to ensure
# good alignment and performant implementation chosen by CuBLAS heuristics.
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
self.pad_size = (-(self.hc_lowrank + self.hc_count)) % 16 if use_combine else 0
if use_combine:
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
self.hyper_hidden_size,
[self.lora_rank, self.hc_count]
[self.hc_lowrank, self.hc_count]
+ ([self.pad_size] if self.pad_size else []),
bias=False,
params_dtype=config.params_dtype,
@@ -117,7 +117,7 @@ class GatedResidual(nn.Module):
else:
self.input_mix_weight_down = ReplicatedLinear(
self.hyper_hidden_size,
self.lora_rank,
self.hc_lowrank,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
@@ -125,7 +125,7 @@ class GatedResidual(nn.Module):
return_bias=False,
)
self.input_mix_weight_up = ReplicatedLinear(
self.lora_rank,
self.hc_lowrank,
self.hyper_hidden_size,
bias=False,
params_dtype=config.params_dtype,
@@ -150,13 +150,13 @@ class GatedResidual(nn.Module):
return hc_down_silu(
xn,
self.input_mix_weight_down_block_inject.weight,
self.lora_rank,
self.hc_lowrank,
self.hc_count,
)
down_and_injection = self.input_mix_weight_down_block_inject(xn)
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
return hc_silu(lora, self.hc_count), injection
split_sizes = [self.hc_lowrank, self.hc_count, self.pad_size]
lowrank, injection, _ = down_and_injection.split(split_sizes, dim=-1)
return hc_silu(lowrank, self.hc_count), injection
def mix(
self, hidden_states: torch.Tensor
@@ -168,8 +168,8 @@ class GatedResidual(nn.Module):
self.hc_count,
)
lora, injection = self._down_and_inject(xn)
gate = self.input_mix_weight_up(lora) # [M, D]
lowrank, injection = self._down_and_inject(xn)
gate = self.input_mix_weight_up(lowrank) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
@@ -196,8 +196,8 @@ class GatedResidual(nn.Module):
self.hc_count,
)
lora, injection = self._down_and_inject(xn)
gate = self.input_mix_weight_up(lora) # [M, D]
lowrank, injection = self._down_and_inject(xn)
gate = self.input_mix_weight_up(lowrank) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.quantization.base_config import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter, sp_shard
from vllm.utils.math_utils import round_up
from ..common.ngram_embedding import (
Qwen4ExpPLEDeviceEmbedding,
@@ -356,7 +357,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
persistent=True,
)
divisor = int(config.make_ngram_vocab_size_divisible_by)
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
padded_vocab_size = round_up(total_vocab_size, divisor)
embedding_prefix = f"{prefix}.ngram_embedding"
embedding_quant_method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
@@ -411,9 +412,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
)
weight = self.ngram_embedding.weight
logger.info(
"Initialized PLE embedding %s: quantization_method=%s, "
"Initialized PLE embedding %s: "
"quantization_config=%s, quantization_method=%s, "
"weight_dtype=%s, weight_device=%s, pinned=%s",
embedding_prefix,
type(quant_config).__name__,
type(embedding_quant_method).__name__,
weight.dtype,
weight.device,
+5 -109
View File
@@ -6,7 +6,6 @@ import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
@triton.jit
@@ -52,7 +51,7 @@ def _grouped_gemma_rmsnorm_kernel(
tl.store(y_ptr + row * stride_y + offsets, y, mask)
def _grouped_gemma_rmsnorm(
def grouped_gemma_rmsnorm(
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
) -> torch.Tensor:
N, DIM = x.shape
@@ -105,7 +104,7 @@ def _hc_silu_kernel(
tl.store(y_ptr + row * stride_y + offs, y, mask)
def _hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
num_tokens, DIM = x.shape
assert x.stride(1) == 1
@@ -160,7 +159,7 @@ def _hc_gate_mix_kernel(
tl.store(y_ptr + row * stride_y + offs_inner, acc, mask)
def _hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
N, DIM = gate.shape
assert x.shape == gate.shape
assert DIM % hc_count == 0
@@ -232,7 +231,7 @@ def _hc_combine_kernel(
tl.store(out_ptr + row * stride_out + offs, out, mask=mask)
def _hc_combine(
def hc_combine(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor | None,
@@ -345,7 +344,7 @@ def _hc_combine_norm_kernel(
tl.store(y_ptr + row * stride_y + offs, y, mask_inner)
def _hc_combine_norm(
def hc_combine_norm(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor | None,
@@ -392,109 +391,6 @@ def _hc_combine_norm(
return out, y
def _same_shape_fake(x: torch.Tensor, *args) -> torch.Tensor:
return x.new_empty(x.shape)
def _hc_gate_mix_fake(
x: torch.Tensor, gate: torch.Tensor, hc_count: int
) -> torch.Tensor:
del gate
return x.new_empty((x.shape[0], x.shape[1] // hc_count))
def _hc_combine_fake(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor | None,
hc_count: int,
) -> torch.Tensor:
del block_output, injection_logits, hc_count
return residual.new_empty(residual.shape)
def _hc_combine_norm_fake(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor | None,
norm_weight: torch.Tensor,
eps: float,
hc_count: int,
) -> tuple[torch.Tensor, torch.Tensor]:
del block_output, injection_logits, norm_weight, eps, hc_count
return residual.new_empty(residual.shape), residual.new_empty(residual.shape)
direct_register_custom_op(
op_name="qwen4_exp_grouped_gemma_rmsnorm",
op_func=_grouped_gemma_rmsnorm,
fake_impl=_same_shape_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_silu",
op_func=_hc_silu,
fake_impl=_same_shape_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_gate_mix",
op_func=_hc_gate_mix,
fake_impl=_hc_gate_mix_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_combine",
op_func=_hc_combine,
fake_impl=_hc_combine_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_combine_norm",
op_func=_hc_combine_norm,
fake_impl=_hc_combine_norm_fake,
)
def grouped_gemma_rmsnorm(
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_grouped_gemma_rmsnorm(x, weight, eps, num_groups)
def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_hc_silu(x, hc_count)
def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_hc_gate_mix(x, gate, hc_count)
def hc_combine(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor | None,
hc_count: int,
) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_hc_combine(
residual, block_output, injection_logits, hc_count
)
def hc_combine_norm(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor | None,
norm_weight: torch.Tensor,
eps: float,
hc_count: int,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.qwen4_exp_hc_combine_norm(
residual,
block_output,
injection_logits,
norm_weight,
eps,
hc_count,
)
__all__ = [
"grouped_gemma_rmsnorm",
"hc_combine",
@@ -10,6 +10,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import round_up
_TOPK_WORKSPACE_BYTES = 1024 * 1024
_DECODE_BLOCK_N = 64
@@ -611,7 +612,7 @@ def qsa_select_paged_prefill(
# No row scores beyond cdiv(max_seq_len, compress_ratio) compressed
# columns. Round up to 64 to keep the logits row stride
# cooperative_topk-compatible.
logits_width = triton.cdiv(triton.cdiv(max_seq_len, compress_ratio), 64) * 64
logits_width = round_up(triton.cdiv(max_seq_len, compress_ratio), 64)
logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1])
# chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB
+29 -3
View File
@@ -38,7 +38,9 @@ def decode_opencv(
int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
)
source = loader_cls._prepare_source(OpenCVVideoBackendMixin.get_video_metadata(cap))
source = loader_cls._prepare_source(
OpenCVVideoBackendMixin.get_video_metadata(data)
)
frame_idx = loader_cls.compute_frames_index_to_sample(
source=source, target=target, **sampling_kwargs
)
@@ -74,10 +76,34 @@ class OpenCVVideoBackendMixin:
raise ValueError("Could not open video stream")
return cap
@classmethod
def demux_count_frames(cls, data: bytes) -> int:
"""Count frames with OpenCV's demuxer-only mode."""
# NOTE: buf must stay referenced until after release().
# OpenCV's Python stream binding does not safely own the stream object
buf = BytesIO(data)
demux_cap = cv2.VideoCapture(
buf, cls.get_cv2_video_api(), [cv2.CAP_PROP_FORMAT, -1]
)
count = 0
while demux_cap.grab():
count += 1
demux_cap.release()
if count == 0:
raise ValueError("Could not determine the number of video frames")
return count
@staticmethod
def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata:
def get_video_metadata(data: bytes) -> VideoSourceMetadata:
cap = OpenCVVideoBackendMixin.open_video_capture(data)
total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
original_fps = cap.get(cv2.CAP_PROP_FPS)
if total_frames_num <= 0:
# Streamed WebM/MKV and raw H.264 miss frame count in headers;
# We need to use demux frames count for retrieval
total_frames_num = OpenCVVideoBackendMixin.demux_count_frames(data)
duration = total_frames_num / original_fps if original_fps > 0 else 0
return VideoSourceMetadata(total_frames_num, original_fps, duration)
# CAP_PROP_FRAME_COUNT counts every physical sample in the container,
# overstating the presentable frames when an mp4 edit list hides the
# decode lead-in (e.g. a lossless `ffmpeg -ss ... -c copy` trim).
@@ -87,7 +113,7 @@ class OpenCVVideoBackendMixin:
# may land mid-stream on corrupt files, so only trust it when the
# stream really ends there. A one-frame tolerance keeps mere header
# miscounts from being "corrected".
if total_frames_num > 0 and cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
if cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
visible_frames = int(cap.get(cv2.CAP_PROP_POS_FRAMES))
at_stream_end = not cap.grab()
cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
+86
View File
@@ -89,6 +89,10 @@ CONFIG_FIELD_ALIASES: dict[str, tuple[str, ...]] = {
`ModelArchConfigConvertorBase` reads a scalar from these; a caller that needs the
raw value (a per-layer `list`, say) passes the aliases to `getattr_iter` itself."""
_ST_TRANSFORMER_MODULE_TYPES = {
"sentence_transformers.models.Transformer",
"sentence_transformers.base.modules.transformer.Transformer",
}
_ST_POOLING_MODULE_TYPES = {
"sentence_transformers.models.Pooling",
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
@@ -1354,6 +1358,88 @@ def try_get_dense_modules(
return None
@cache
def get_sentence_transformers_chat_template_kwargs(
model: str | Path,
revision: str | None = None,
) -> dict[str, Any] | None:
"""Get the saved chat template settings of a Sentence Transformers
CrossEncoder whose single Transformer module declares the `message`
modality.
Sentence Transformers formats the pairs of such checkpoints as
`query`/`document` messages with the saved chat template.
Args:
model: Local checkpoint path or Hugging Face model ID.
revision: Model revision to inspect.
Returns:
The saved `processing_kwargs.chat_template` (possibly empty), or
`None` if the checkpoint does not declare the `message` modality.
Raises:
ValueError: If the saved chat template settings are unsupported.
"""
try:
st_config = get_hf_file_to_dict(
"config_sentence_transformers.json", model, revision
)
if not isinstance(st_config, dict) or (
st_config.get("model_type") != "CrossEncoder"
):
return None
modules = get_hf_file_to_dict("modules.json", model, revision)
transformer_config = get_hf_file_to_dict(
"sentence_bert_config.json", model, revision
)
except Exception:
logger.debug("Failed to read the Sentence Transformers config", exc_info=True)
return None
if isinstance(modules, dict):
modules = modules.get("modules")
if not (
isinstance(modules, list)
and len(modules) == 1
and isinstance(modules[0], dict)
and modules[0].get("type") in _ST_TRANSFORMER_MODULE_TYPES
and not modules[0].get("path")
and isinstance(transformer_config, dict)
and transformer_config.get("transformer_task") == "sequence-classification"
and isinstance(
modality_config := transformer_config.get("modality_config"), dict
)
and "message" in modality_config
):
return None
processing_kwargs = transformer_config.get("processing_kwargs") or {}
chat_template_kwargs = (
processing_kwargs.get("chat_template") or {}
if isinstance(processing_kwargs, dict)
else None
)
if not isinstance(chat_template_kwargs, dict):
raise ValueError(
"The Sentence Transformers processing_kwargs.chat_template must be "
"a JSON object."
)
chat_template_kwargs = dict(chat_template_kwargs)
# vLLM does not restore the template suffix after truncation.
chat_template_kwargs.pop("restore_suffix", None)
if not isinstance(chat_template_kwargs.get("chat_template", ""), str):
raise ValueError(
"The Sentence Transformers chat template name must be a string."
)
if reserved := {"tokenize", "tools"} & chat_template_kwargs.keys():
raise ValueError(
f"Unsupported Sentence Transformers chat template settings: {reserved}"
)
return chat_template_kwargs
def _read_safetensors_metadata_in_dir(local_dir: Path) -> dict[str, Any]:
return {
param_name: info