Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -27,6 +27,7 @@ from .utils import (
|
||||
BeamSearchOutput,
|
||||
BeamSearchSequence,
|
||||
create_sort_beams_key_function,
|
||||
get_beam_search_eos_token_ids,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -86,8 +87,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
self.llm_engine.input_processor.resolve_watermarking(params)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
eos_token_id = tokenizer.eos_token_id
|
||||
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
|
||||
|
||||
engine_inputs = self._preprocess_cmpl(prompts)
|
||||
lora_requests = self._lora_request_to_seq(lora_request, len(engine_inputs))
|
||||
@@ -129,6 +128,11 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
params.structured_outputs, tokenizer
|
||||
)
|
||||
|
||||
eos_token_ids = get_beam_search_eos_token_ids(
|
||||
self.llm_engine.input_processor.generation_config_fields,
|
||||
tokenizer.eos_token_id,
|
||||
)
|
||||
sort_beams_key = create_sort_beams_key_function(eos_token_ids, length_penalty)
|
||||
instances: list[BeamSearchInstance] = []
|
||||
|
||||
for lora_req, prompt in zip(lora_requests, engine_inputs):
|
||||
@@ -169,7 +173,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
should_stop = self._beam_search_step(
|
||||
instances_batch=instances_batch,
|
||||
base_sampling_params=base_sampling_params,
|
||||
eos_token_id=eos_token_id,
|
||||
eos_token_ids=eos_token_ids,
|
||||
ignore_eos=ignore_eos,
|
||||
beam_width=beam_width,
|
||||
sort_beams_key=sort_beams_key,
|
||||
@@ -204,7 +208,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
self,
|
||||
instances_batch: list[BeamSearchInstance],
|
||||
base_sampling_params: SamplingParams,
|
||||
eos_token_id: int | None,
|
||||
eos_token_ids: set[int],
|
||||
ignore_eos: bool,
|
||||
beam_width: int,
|
||||
sort_beams_key: Callable,
|
||||
@@ -350,7 +354,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
cum_logprob=current_beam.cum_logprob + logprob_obj.logprob,
|
||||
)
|
||||
|
||||
if token_id == eos_token_id and not ignore_eos:
|
||||
if token_id in eos_token_ids and not ignore_eos:
|
||||
instance.completed.append(new_beam)
|
||||
else:
|
||||
instance_new_beams.append(new_beam)
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.utils.async_utils import collect_from_async_generator
|
||||
from .utils import (
|
||||
BeamSearchSequence,
|
||||
create_sort_beams_key_function,
|
||||
get_beam_search_eos_token_ids,
|
||||
)
|
||||
|
||||
|
||||
@@ -47,8 +48,6 @@ class BeamSearchOnlineMixin(ABC):
|
||||
self.engine_client.input_processor.resolve_watermarking(params)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
eos_token_id = tokenizer.eos_token_id
|
||||
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
|
||||
|
||||
if prompt["type"] == "embeds":
|
||||
raise NotImplementedError("Embedding prompt not supported for beam search")
|
||||
@@ -74,6 +73,11 @@ class BeamSearchOnlineMixin(ABC):
|
||||
sampling_params, self.engine_client.input_processor.vllm_config
|
||||
)
|
||||
allowed = set(params.allowed_token_ids) if params.allowed_token_ids else None
|
||||
eos_token_ids = get_beam_search_eos_token_ids(
|
||||
self.engine_client.input_processor.generation_config_fields,
|
||||
tokenizer.eos_token_id,
|
||||
)
|
||||
sort_beams_key = create_sort_beams_key_function(eos_token_ids, length_penalty)
|
||||
all_beams = [
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
@@ -159,17 +163,17 @@ class BeamSearchOnlineMixin(ABC):
|
||||
candidate_logprob = (
|
||||
current_beam.cum_logprob + logprob_obj.logprob
|
||||
)
|
||||
if token_id == eos_token_id and not ignore_eos:
|
||||
if token_id in eos_token_ids and not ignore_eos:
|
||||
completed.append(
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
tokens=current_beam.tokens + [eos_token_id]
|
||||
tokens=current_beam.tokens + [token_id]
|
||||
if include_stop_str_in_output
|
||||
else current_beam.tokens,
|
||||
logprobs=current_beam.logprobs + [logprobs],
|
||||
cum_logprob=candidate_logprob,
|
||||
finish_reason="stop",
|
||||
stop_reason=eos_token_id,
|
||||
stop_reason=token_id,
|
||||
)
|
||||
)
|
||||
else:
|
||||
@@ -219,7 +223,7 @@ class BeamSearchOnlineMixin(ABC):
|
||||
best_beams = sorted_completed[:beam_width]
|
||||
|
||||
for beam in best_beams:
|
||||
if beam.tokens[-1] == eos_token_id and not ignore_eos:
|
||||
if beam.tokens[-1] in eos_token_ids and not ignore_eos:
|
||||
# Skip the eos token in the text.
|
||||
tokens = beam.tokens[tokenized_length:-1]
|
||||
else:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.inputs import (
|
||||
DecoderOnlyEngineInput,
|
||||
@@ -13,6 +14,7 @@ from vllm.inputs import (
|
||||
)
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -137,7 +139,7 @@ class BeamSearchInstance:
|
||||
def get_beam_search_score(
|
||||
tokens: list[int],
|
||||
cumulative_logprob: float,
|
||||
eos_token_id: int,
|
||||
eos_token_ids: set[int],
|
||||
length_penalty: float = 1.0,
|
||||
) -> float:
|
||||
"""Calculate the beam search score with length penalty.
|
||||
@@ -147,17 +149,31 @@ def get_beam_search_score(
|
||||
https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
|
||||
"""
|
||||
seq_len = len(tokens)
|
||||
if tokens[-1] == eos_token_id:
|
||||
if tokens[-1] in eos_token_ids:
|
||||
seq_len -= 1
|
||||
|
||||
# An aborted beam may contain only an EOS prompt token.
|
||||
return cumulative_logprob / (max(seq_len, 1) ** length_penalty)
|
||||
|
||||
|
||||
def create_sort_beams_key_function(eos_token_id: int, length_penalty: float):
|
||||
def create_sort_beams_key_function(eos_token_ids: set[int], length_penalty: float):
|
||||
def sort_beams_key(x: BeamSearchSequence) -> float:
|
||||
return get_beam_search_score(
|
||||
x.tokens, x.cum_logprob, eos_token_id, length_penalty
|
||||
x.tokens, x.cum_logprob, eos_token_ids, length_penalty
|
||||
)
|
||||
|
||||
return sort_beams_key
|
||||
|
||||
|
||||
def get_beam_search_eos_token_ids(
|
||||
generation_config: dict[str, Any], eos_token_id: int | None
|
||||
) -> set[int]:
|
||||
"""Return the token ids that end generation: the tokenizer EOS plus the
|
||||
generation config's ``eos_token_id``.
|
||||
|
||||
This uses a separate SamplingParams so the params sent to the engine keep
|
||||
their own ``stop_token_ids`` and are validated as before.
|
||||
"""
|
||||
params = SamplingParams()
|
||||
params.update_from_generation_config(generation_config, eos_token_id)
|
||||
return params.all_stop_token_ids
|
||||
|
||||
@@ -9,14 +9,17 @@ import torch.nn.functional as F
|
||||
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
|
||||
from vllm.exceptions import ChatTemplateResolutionError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.hf import safe_apply_chat_template
|
||||
from vllm.renderers import TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.hf import resolve_chat_template, safe_apply_chat_template
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_target_prompt,
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.transformers_utils.config import (
|
||||
get_sentence_transformers_chat_template_kwargs,
|
||||
)
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
@@ -457,9 +460,22 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
self.model = model if self.supports_score_template else None
|
||||
self.use_sep_token = self.model_config.use_sep_token
|
||||
|
||||
self.saved_chat_template: str | None = None
|
||||
self.saved_chat_template_kwargs: dict[str, Any] = {}
|
||||
if not self.supports_score_template:
|
||||
try:
|
||||
self.saved_chat_template, self.saved_chat_template_kwargs = (
|
||||
self._get_saved_chat_template()
|
||||
)
|
||||
except ValueError:
|
||||
# An explicit chat template takes precedence anyway.
|
||||
if self.chat_template is None:
|
||||
raise
|
||||
|
||||
if (
|
||||
getattr(self.model_config.hf_config, "is_original_qwen3_reranker", False)
|
||||
and self.chat_template is None
|
||||
and self.saved_chat_template is None
|
||||
):
|
||||
suggested_template = (
|
||||
"examples/pooling/score/template/qwen3_vl_reranker.jinja"
|
||||
@@ -475,6 +491,41 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
suggested_template,
|
||||
)
|
||||
|
||||
def _get_saved_chat_template(self) -> tuple[str | None, dict[str, Any]]:
|
||||
"""Get the chat template and kwargs declared by the Sentence
|
||||
Transformers config, if any."""
|
||||
saved_kwargs = get_sentence_transformers_chat_template_kwargs(
|
||||
self.model_config.model, self.model_config.revision
|
||||
)
|
||||
if saved_kwargs is None:
|
||||
return None, {}
|
||||
|
||||
saved_kwargs = dict(saved_kwargs)
|
||||
name = saved_kwargs.pop("chat_template", None)
|
||||
if name == "default":
|
||||
name = None
|
||||
template = resolve_chat_template(
|
||||
self.tokenizer,
|
||||
chat_template=name,
|
||||
tools=None,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
# Hugging Face returns an unknown template name unchanged.
|
||||
if template is None or template == name:
|
||||
raise ValueError(
|
||||
"The Sentence Transformers config declares message inputs, but "
|
||||
f"the checkpoint has no {name or 'default'!r} chat template."
|
||||
)
|
||||
|
||||
if self.chat_template is None:
|
||||
logger.info_once(
|
||||
"Scoring with the %r chat template declared by the Sentence "
|
||||
"Transformers config. Its suffix is not restored after "
|
||||
"truncation; use max_tokens_per_doc to truncate documents.",
|
||||
name or "default",
|
||||
)
|
||||
return template, saved_kwargs
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
@@ -648,6 +699,12 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
model_config = self.model_config
|
||||
tokenizer = self.tokenizer
|
||||
|
||||
if chat_template is None and self.saved_chat_template is not None:
|
||||
chat_template = self.saved_chat_template
|
||||
chat_template_kwargs = merge_kwargs(
|
||||
self.saved_chat_template_kwargs, chat_template_kwargs
|
||||
)
|
||||
|
||||
prompt_1, prompt_2, mm_data, mm_uuids = parse_score_data(
|
||||
data_1,
|
||||
data_2,
|
||||
@@ -703,7 +760,8 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
prompt_inputs = tokenizer(text=full_prompt, **local_kwargs)
|
||||
return full_prompt, prompt_inputs
|
||||
|
||||
# FIXME: For now, we only apply a template when one is explicitly provided.
|
||||
# FIXME: For now, we only apply a template when one is explicitly provided,
|
||||
# either directly or by the Sentence Transformers config.
|
||||
# We cannot rely on the tokenizer's chat template because many models
|
||||
# inherit junk templates from their base LLM, which breaks both the models
|
||||
# and the tests that use them.
|
||||
|
||||
@@ -82,6 +82,8 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
|
||||
from vllm.utils.torch_utils import aux_stream
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_token_offset
|
||||
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
@@ -713,6 +715,18 @@ class DeepseekV4MoE(nn.Module):
|
||||
),
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
self.shared_experts_stream = (
|
||||
aux_stream()
|
||||
if self.shared_experts is not None
|
||||
and not self.experts.fuse_shared_expert
|
||||
and not envs.VLLM_DISABLE_SHARED_EXPERTS_STREAM
|
||||
else None
|
||||
)
|
||||
self.shared_experts_events = (
|
||||
(torch.cuda.Event(), torch.cuda.Event())
|
||||
if self.shared_experts_stream is not None
|
||||
else None
|
||||
)
|
||||
return
|
||||
|
||||
fuse_shared_into_routed = (
|
||||
@@ -777,6 +791,30 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
def _forward_mega_moe(
|
||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
shared_experts = self.shared_experts
|
||||
if shared_experts is None:
|
||||
return self._forward_mega_moe_routed(hidden_states, input_ids)
|
||||
events = self.shared_experts_events
|
||||
if (
|
||||
events is None
|
||||
or hidden_states.shape[0] > envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
||||
):
|
||||
return self._forward_mega_moe_routed(
|
||||
hidden_states, input_ids
|
||||
) + shared_experts(hidden_states)
|
||||
# The unfused shared expert overlaps router + MegaMoE in small steps.
|
||||
routed, shared = maybe_execute_in_parallel(
|
||||
fn0=lambda: self._forward_mega_moe_routed(hidden_states, input_ids),
|
||||
fn1=lambda: shared_experts(hidden_states),
|
||||
event0=events[0],
|
||||
event1=events[1],
|
||||
aux_stream=self.shared_experts_stream,
|
||||
)
|
||||
return routed + shared
|
||||
|
||||
def _forward_mega_moe_routed(
|
||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
@@ -792,12 +830,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
bias_vl=self.gate.bias_vl,
|
||||
image_sentinel_lo=self.image_sentinel_lo,
|
||||
)
|
||||
final_hidden_states = self.experts(hidden_states, topk_weights, topk_ids)
|
||||
if self.shared_experts is not None:
|
||||
final_hidden_states = final_hidden_states + self.shared_experts(
|
||||
hidden_states
|
||||
)
|
||||
return final_hidden_states
|
||||
return self.experts(hidden_states, topk_weights, topk_ids)
|
||||
|
||||
|
||||
# Hidden sizes supported by AITER mhc_pre_big_fuse_rmsnorm.
|
||||
|
||||
@@ -70,7 +70,7 @@ class GatedResidual(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.lora_rank = config.hc_lowrank
|
||||
self.hc_lowrank = config.hc_lowrank
|
||||
self.hc_count = config.hc_count
|
||||
self.hidden_size = config.hidden_size
|
||||
self.use_combine = use_combine
|
||||
@@ -91,11 +91,11 @@ class GatedResidual(nn.Module):
|
||||
# -- vLLM Linear weights --------------------------------------------
|
||||
# The merged skinny-GEMM shape is physically padded to 16 rows for
|
||||
# alignment and efficient backend dispatch.
|
||||
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
|
||||
self.pad_size = (-(self.hc_lowrank + self.hc_count)) % 16 if use_combine else 0
|
||||
if use_combine:
|
||||
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
|
||||
self.hyper_hidden_size,
|
||||
[self.lora_rank, self.hc_count]
|
||||
[self.hc_lowrank, self.hc_count]
|
||||
+ ([self.pad_size] if self.pad_size else []),
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
@@ -107,7 +107,7 @@ class GatedResidual(nn.Module):
|
||||
else:
|
||||
self.input_mix_weight_down = ReplicatedLinear(
|
||||
self.hyper_hidden_size,
|
||||
self.lora_rank,
|
||||
self.hc_lowrank,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
@@ -115,7 +115,7 @@ class GatedResidual(nn.Module):
|
||||
return_bias=False,
|
||||
)
|
||||
self.input_mix_weight_up = ReplicatedLinear(
|
||||
self.lora_rank,
|
||||
self.hc_lowrank,
|
||||
self.hyper_hidden_size,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
@@ -136,15 +136,15 @@ class GatedResidual(nn.Module):
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
split_sizes = [self.hc_lowrank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
lowrank, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
lowrank = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
lowrank = hc_silu(lowrank, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lowrank) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
@@ -172,15 +172,15 @@ class GatedResidual(nn.Module):
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
split_sizes = [self.hc_lowrank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
lowrank, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
lowrank = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
lowrank = hc_silu(lowrank, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lowrank) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
@@ -242,7 +243,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
persistent=True,
|
||||
)
|
||||
divisor = int(config.make_ngram_vocab_size_divisible_by)
|
||||
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
|
||||
padded_vocab_size = round_up(total_vocab_size, divisor)
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
embedding_prefix = f"{prefix}.ngram_embedding"
|
||||
@@ -280,9 +281,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
reduce_results=not self.use_reduce_scatter,
|
||||
)
|
||||
logger.info(
|
||||
"Initialized AMD PLE embedding %s: quantization_method=%s, "
|
||||
"Initialized AMD PLE embedding %s: "
|
||||
"quantization_config=%s, quantization_method=%s, "
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s",
|
||||
embedding_prefix,
|
||||
type(quant_config).__name__,
|
||||
type(embedding_quant_method).__name__,
|
||||
self.ngram_embedding.weight.dtype,
|
||||
self.ngram_embedding.weight.device,
|
||||
@@ -742,7 +745,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
)
|
||||
kv, _ = self.kv_proj(embeddings)
|
||||
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
|
||||
gated_output, conv_input = ple_gate(
|
||||
gated_output, conv_input = torch.ops.vllm.qwen4_exp_ple_gate(
|
||||
key,
|
||||
value,
|
||||
hidden_states,
|
||||
@@ -848,6 +851,28 @@ def qwen4_exp_ple_short_conv(
|
||||
layer._short_conv(inputs, residual, outer_residual)
|
||||
|
||||
|
||||
def _ple_gate_fake(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
hidden: torch.Tensor,
|
||||
norm_key_w: torch.Tensor,
|
||||
norm_query_w: torch.Tensor,
|
||||
norm_conv_w: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return the gate output shapes for AMD graph tracing."""
|
||||
return torch.empty_like(hidden), torch.empty_like(hidden)
|
||||
|
||||
|
||||
# Keep the AMD gate opaque to Dynamo while reusing the shared Triton kernel.
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_gate",
|
||||
op_func=ple_gate,
|
||||
mutates_args=[],
|
||||
fake_impl=_ple_gate_fake,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_amd_ple_ngram_ids",
|
||||
op_func=qwen4_exp_amd_ple_ngram_ids,
|
||||
|
||||
@@ -194,22 +194,21 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
||||
)
|
||||
if quant_config is None:
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
|
||||
quant_algo = quant_config._resolve_quant_algo(prefix)
|
||||
if quant_algo == "FP8":
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
if quant_algo == "NVFP4":
|
||||
return Qwen4ExpPLENvFp4EmbeddingMethod()
|
||||
if quant_algo is not None:
|
||||
raise ValueError(
|
||||
f"Unsupported quant_algo {quant_algo!r} for the PLE embedding "
|
||||
f"{prefix!r}; expected FP8 or NVFP4"
|
||||
)
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if isinstance(
|
||||
quant_config, ModelOptQuantConfigBase
|
||||
) and quant_config.is_layer_excluded(prefix):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if isinstance(quant_config, ModelOptQuantConfigBase):
|
||||
if quant_config.is_layer_excluded(prefix):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
|
||||
quant_algo = quant_config._resolve_quant_algo(prefix)
|
||||
if quant_algo == "FP8":
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
if quant_algo == "NVFP4":
|
||||
return Qwen4ExpPLENvFp4EmbeddingMethod()
|
||||
if quant_algo is not None:
|
||||
raise ValueError(
|
||||
f"Unsupported quant_algo {quant_algo!r} for the PLE embedding "
|
||||
f"{prefix!r}; expected FP8 or NVFP4"
|
||||
)
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if (
|
||||
isinstance(quant_config, CompressedTensorsConfig)
|
||||
and quant_config.get_scheme_dict(None, layer_name=prefix) is None
|
||||
@@ -223,29 +222,32 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
||||
# Quark quantizes only Linear and MoE layers; PLE tables stay BF16.
|
||||
if isinstance(quant_config, QuarkConfig):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if not isinstance(quant_config, Fp8Config):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp PLE embedding does not support quantization config "
|
||||
f"{type(quant_config).__name__}"
|
||||
)
|
||||
|
||||
ignored_layers = quant_config.ignored_layers
|
||||
if is_layer_skipped(
|
||||
prefix,
|
||||
ignored_layers,
|
||||
quant_config.packed_modules_mapping,
|
||||
match_mode=quant_config.ignored_layers_match_mode,
|
||||
):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
# PLE checkpoint shards form one runtime embedding parameter.
|
||||
shard_prefix = f"{prefix}.shard_"
|
||||
if any(name.startswith(shard_prefix) for name in ignored_layers):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if not quant_config.is_checkpoint_fp8_serialized:
|
||||
if isinstance(quant_config, Fp8Config):
|
||||
ignored_layers = quant_config.ignored_layers
|
||||
if is_layer_skipped(
|
||||
prefix,
|
||||
ignored_layers,
|
||||
quant_config.packed_modules_mapping,
|
||||
match_mode=quant_config.ignored_layers_match_mode,
|
||||
):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
# PLE checkpoint shards form one runtime embedding parameter.
|
||||
shard_prefix = f"{prefix}.shard_"
|
||||
if any(name.startswith(shard_prefix) for name in ignored_layers):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if quant_config.is_checkpoint_fp8_serialized:
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp PLE embedding only supports serialized FP8 checkpoints"
|
||||
)
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
|
||||
logger.warning(
|
||||
"Falling back to unquantized PLE embedding for %s "
|
||||
"with quantization config %s.",
|
||||
prefix,
|
||||
type(quant_config).__name__,
|
||||
)
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -781,6 +783,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
)
|
||||
active_output = buffer[: gathered_ids.shape[0]]
|
||||
prefetch_stream.wait_stream(torch.cuda.current_stream())
|
||||
# Prevent ID storage reuse until the prefetch stream finishes reading it.
|
||||
gathered_ids.record_stream(prefetch_stream)
|
||||
with torch.cuda.stream(prefetch_stream):
|
||||
self._lookup(gathered_ids, output=active_output)
|
||||
|
||||
@@ -16,7 +16,6 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
@@ -240,7 +239,7 @@ def _ple_gate_kernel(
|
||||
tl.store(normed_ptr + t * HC * H + offs, normed, mask=mask)
|
||||
|
||||
|
||||
def _ple_gate(
|
||||
def ple_gate(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
hidden: torch.Tensor,
|
||||
@@ -281,40 +280,6 @@ def _ple_gate(
|
||||
return gated, normed
|
||||
|
||||
|
||||
def _ple_gate_fake(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
hidden: torch.Tensor,
|
||||
norm_key_w: torch.Tensor,
|
||||
norm_query_w: torch.Tensor,
|
||||
norm_conv_w: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.empty_like(hidden), torch.empty_like(hidden)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_gate",
|
||||
op_func=_ple_gate,
|
||||
mutates_args=[],
|
||||
fake_impl=_ple_gate_fake,
|
||||
)
|
||||
|
||||
|
||||
def ple_gate(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
hidden: torch.Tensor,
|
||||
norm_key_w: torch.Tensor,
|
||||
norm_query_w: torch.Tensor,
|
||||
norm_conv_w: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.qwen4_exp_ple_gate(
|
||||
key, value, hidden, norm_key_w, norm_query_w, norm_conv_w, eps
|
||||
)
|
||||
|
||||
|
||||
# Dilated short convolution
|
||||
@triton.jit(do_not_specialize=["num_reqs", "bs_iters", "has_token_map"])
|
||||
def _ple_conv_kernel(
|
||||
|
||||
@@ -20,66 +20,64 @@ class PLEShardOverlap:
|
||||
row_count: int
|
||||
|
||||
|
||||
def compute_ple_shard_overlap(
|
||||
*,
|
||||
checkpoint_start: int,
|
||||
checkpoint_rows: int,
|
||||
tp_start: int,
|
||||
tp_end: int,
|
||||
) -> PLEShardOverlap | None:
|
||||
"""Compute the overlap of a checkpoint shard and one TP vocabulary range."""
|
||||
if checkpoint_start < 0 or checkpoint_rows < 0:
|
||||
raise ValueError("checkpoint shard bounds must be non-negative")
|
||||
if tp_start < 0 or tp_end < tp_start:
|
||||
raise ValueError("invalid TP vocabulary range")
|
||||
checkpoint_end = checkpoint_start + checkpoint_rows
|
||||
overlap_start = max(checkpoint_start, tp_start)
|
||||
overlap_end = min(checkpoint_end, tp_end)
|
||||
if overlap_start >= overlap_end:
|
||||
return None
|
||||
return PLEShardOverlap(
|
||||
source_start=overlap_start - checkpoint_start,
|
||||
destination_start=overlap_start - tp_start,
|
||||
row_count=overlap_end - overlap_start,
|
||||
)
|
||||
|
||||
|
||||
def copy_ple_embedding_shard_(
|
||||
destination: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
*,
|
||||
checkpoint_start: int,
|
||||
tp_start: int,
|
||||
tp_end: int,
|
||||
) -> int:
|
||||
"""Copy the overlapping rows of a PLE checkpoint shard into a TP table."""
|
||||
if destination.ndim == 0 or loaded_weight.ndim != destination.ndim:
|
||||
raise ValueError("destination and loaded weight must have matching ranks")
|
||||
if destination.shape[1:] != loaded_weight.shape[1:]:
|
||||
raise ValueError(
|
||||
"embedding shard dimensions do not match: "
|
||||
f"{tuple(destination.shape[1:])} != {tuple(loaded_weight.shape[1:])}"
|
||||
)
|
||||
if destination.shape[0] < tp_end - tp_start:
|
||||
raise ValueError("destination does not cover the requested TP range")
|
||||
overlap = compute_ple_shard_overlap(
|
||||
checkpoint_start=checkpoint_start,
|
||||
checkpoint_rows=loaded_weight.shape[0],
|
||||
tp_start=tp_start,
|
||||
tp_end=tp_end,
|
||||
)
|
||||
if overlap is None:
|
||||
return 0
|
||||
source = loaded_weight.narrow(0, overlap.source_start, overlap.row_count)
|
||||
target = destination.narrow(0, overlap.destination_start, overlap.row_count)
|
||||
with torch.no_grad():
|
||||
target.copy_(source.to(device=target.device, dtype=target.dtype))
|
||||
return overlap.row_count
|
||||
|
||||
|
||||
class PLEVocabParallelEmbedding(VocabParallelEmbedding):
|
||||
"""Vocab-parallel embedding that accepts checkpoint row shards."""
|
||||
|
||||
def _compute_shard_overlap(
|
||||
self,
|
||||
*,
|
||||
checkpoint_start: int,
|
||||
checkpoint_rows: int,
|
||||
) -> PLEShardOverlap | None:
|
||||
"""Compute the overlap of a checkpoint shard and one TP vocabulary range."""
|
||||
tp_start = self.shard_indices.org_vocab_start_index
|
||||
tp_end = self.shard_indices.org_vocab_end_index
|
||||
if checkpoint_start < 0 or checkpoint_rows < 0:
|
||||
raise ValueError("checkpoint shard bounds must be non-negative")
|
||||
if tp_start < 0 or tp_end < tp_start:
|
||||
raise ValueError("invalid TP vocabulary range")
|
||||
checkpoint_end = checkpoint_start + checkpoint_rows
|
||||
overlap_start = max(checkpoint_start, tp_start)
|
||||
overlap_end = min(checkpoint_end, tp_end)
|
||||
if overlap_start >= overlap_end:
|
||||
return None
|
||||
return PLEShardOverlap(
|
||||
source_start=overlap_start - checkpoint_start,
|
||||
destination_start=overlap_start - tp_start,
|
||||
row_count=overlap_end - overlap_start,
|
||||
)
|
||||
|
||||
def _copy_embedding_shard_(
|
||||
self,
|
||||
destination: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
*,
|
||||
checkpoint_start: int,
|
||||
) -> int:
|
||||
"""Copy the overlapping rows of a PLE checkpoint shard into a TP table."""
|
||||
tp_start = self.shard_indices.org_vocab_start_index
|
||||
tp_end = self.shard_indices.org_vocab_end_index
|
||||
if destination.ndim == 0 or loaded_weight.ndim != destination.ndim:
|
||||
raise ValueError("destination and loaded weight must have matching ranks")
|
||||
if destination.shape[1:] != loaded_weight.shape[1:]:
|
||||
raise ValueError(
|
||||
"embedding shard dimensions do not match: "
|
||||
f"{tuple(destination.shape[1:])} != {tuple(loaded_weight.shape[1:])}"
|
||||
)
|
||||
if destination.shape[0] < tp_end - tp_start:
|
||||
raise ValueError("destination does not cover the requested TP range")
|
||||
overlap = self._compute_shard_overlap(
|
||||
checkpoint_start=checkpoint_start,
|
||||
checkpoint_rows=loaded_weight.shape[0],
|
||||
)
|
||||
if overlap is None:
|
||||
return 0
|
||||
source = loaded_weight.narrow(0, overlap.source_start, overlap.row_count)
|
||||
target = destination.narrow(0, overlap.destination_start, overlap.row_count)
|
||||
with torch.no_grad():
|
||||
target.copy_(source.to(device=target.device, dtype=target.dtype))
|
||||
return overlap.row_count
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: torch.Tensor,
|
||||
@@ -89,10 +87,8 @@ class PLEVocabParallelEmbedding(VocabParallelEmbedding):
|
||||
if checkpoint_start is None:
|
||||
super().weight_loader(param, loaded_weight)
|
||||
return
|
||||
copy_ple_embedding_shard_(
|
||||
self._copy_embedding_shard_(
|
||||
param,
|
||||
loaded_weight,
|
||||
checkpoint_start=checkpoint_start,
|
||||
tp_start=self.shard_indices.org_vocab_start_index,
|
||||
tp_end=self.shard_indices.org_vocab_end_index,
|
||||
)
|
||||
|
||||
@@ -24,7 +24,7 @@ from vllm.config.cache import CacheDType
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -880,9 +880,7 @@ class QSAKeyStateCache(_QSAStateCache):
|
||||
key_head_size = int(kwargs.pop("head_size"))
|
||||
self.key_head_size = key_head_size
|
||||
self.cache_rope_positions = bool(cache_rope_positions)
|
||||
self.rope_position_offset = (
|
||||
(key_head_size + self._BF16_PER_INT64 - 1) // self._BF16_PER_INT64
|
||||
) * self._BF16_PER_INT64
|
||||
self.rope_position_offset = round_up(key_head_size, self._BF16_PER_INT64)
|
||||
storage_head_size = key_head_size
|
||||
if self.cache_rope_positions:
|
||||
storage_head_size = self.rope_position_offset + (
|
||||
@@ -910,7 +908,7 @@ class QSAKeyStateCache(_QSAStateCache):
|
||||
# the scheduler block size). Anything narrower lets a rejected draft row
|
||||
# overwrite a committed key the next step needs to close the group.
|
||||
span = self.compress_ratio + vllm_config.num_speculative_tokens
|
||||
capacity = self.compress_ratio * cdiv(span, self.compress_ratio)
|
||||
capacity = round_up(span, self.compress_ratio)
|
||||
assert self.cache_config.block_size % capacity == 0, (
|
||||
f"QSA ring capacity {capacity} must divide the attention block "
|
||||
f"size {self.cache_config.block_size}"
|
||||
|
||||
@@ -74,7 +74,7 @@ class GatedResidual(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.lora_rank = config.hc_lowrank
|
||||
self.hc_lowrank = config.hc_lowrank
|
||||
self.hc_count = config.hc_count
|
||||
self.hidden_size = config.hidden_size
|
||||
self.use_combine = use_combine
|
||||
@@ -95,11 +95,11 @@ class GatedResidual(nn.Module):
|
||||
# -- vLLM Linear weights --------------------------------------------
|
||||
# The merged skinny-GEMM shape is physically padded to 16 rows to ensure
|
||||
# good alignment and performant implementation chosen by CuBLAS heuristics.
|
||||
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
|
||||
self.pad_size = (-(self.hc_lowrank + self.hc_count)) % 16 if use_combine else 0
|
||||
if use_combine:
|
||||
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
|
||||
self.hyper_hidden_size,
|
||||
[self.lora_rank, self.hc_count]
|
||||
[self.hc_lowrank, self.hc_count]
|
||||
+ ([self.pad_size] if self.pad_size else []),
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
@@ -117,7 +117,7 @@ class GatedResidual(nn.Module):
|
||||
else:
|
||||
self.input_mix_weight_down = ReplicatedLinear(
|
||||
self.hyper_hidden_size,
|
||||
self.lora_rank,
|
||||
self.hc_lowrank,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
@@ -125,7 +125,7 @@ class GatedResidual(nn.Module):
|
||||
return_bias=False,
|
||||
)
|
||||
self.input_mix_weight_up = ReplicatedLinear(
|
||||
self.lora_rank,
|
||||
self.hc_lowrank,
|
||||
self.hyper_hidden_size,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
@@ -150,13 +150,13 @@ class GatedResidual(nn.Module):
|
||||
return hc_down_silu(
|
||||
xn,
|
||||
self.input_mix_weight_down_block_inject.weight,
|
||||
self.lora_rank,
|
||||
self.hc_lowrank,
|
||||
self.hc_count,
|
||||
)
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
return hc_silu(lora, self.hc_count), injection
|
||||
split_sizes = [self.hc_lowrank, self.hc_count, self.pad_size]
|
||||
lowrank, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
return hc_silu(lowrank, self.hc_count), injection
|
||||
|
||||
def mix(
|
||||
self, hidden_states: torch.Tensor
|
||||
@@ -168,8 +168,8 @@ class GatedResidual(nn.Module):
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
lora, injection = self._down_and_inject(xn)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
lowrank, injection = self._down_and_inject(xn)
|
||||
gate = self.input_mix_weight_up(lowrank) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
@@ -196,8 +196,8 @@ class GatedResidual(nn.Module):
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
lora, injection = self._down_and_inject(xn)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
lowrank, injection = self._down_and_inject(xn)
|
||||
gate = self.input_mix_weight_up(lowrank) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter, sp_shard
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
from ..common.ngram_embedding import (
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
@@ -356,7 +357,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
persistent=True,
|
||||
)
|
||||
divisor = int(config.make_ngram_vocab_size_divisible_by)
|
||||
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
|
||||
padded_vocab_size = round_up(total_vocab_size, divisor)
|
||||
embedding_prefix = f"{prefix}.ngram_embedding"
|
||||
embedding_quant_method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
@@ -411,9 +412,11 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
)
|
||||
weight = self.ngram_embedding.weight
|
||||
logger.info(
|
||||
"Initialized PLE embedding %s: quantization_method=%s, "
|
||||
"Initialized PLE embedding %s: "
|
||||
"quantization_config=%s, quantization_method=%s, "
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s",
|
||||
embedding_prefix,
|
||||
type(quant_config).__name__,
|
||||
type(embedding_quant_method).__name__,
|
||||
weight.dtype,
|
||||
weight.device,
|
||||
|
||||
@@ -6,7 +6,6 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -52,7 +51,7 @@ def _grouped_gemma_rmsnorm_kernel(
|
||||
tl.store(y_ptr + row * stride_y + offsets, y, mask)
|
||||
|
||||
|
||||
def _grouped_gemma_rmsnorm(
|
||||
def grouped_gemma_rmsnorm(
|
||||
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
|
||||
) -> torch.Tensor:
|
||||
N, DIM = x.shape
|
||||
@@ -105,7 +104,7 @@ def _hc_silu_kernel(
|
||||
tl.store(y_ptr + row * stride_y + offs, y, mask)
|
||||
|
||||
|
||||
def _hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
num_tokens, DIM = x.shape
|
||||
assert x.stride(1) == 1
|
||||
|
||||
@@ -160,7 +159,7 @@ def _hc_gate_mix_kernel(
|
||||
tl.store(y_ptr + row * stride_y + offs_inner, acc, mask)
|
||||
|
||||
|
||||
def _hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
N, DIM = gate.shape
|
||||
assert x.shape == gate.shape
|
||||
assert DIM % hc_count == 0
|
||||
@@ -232,7 +231,7 @@ def _hc_combine_kernel(
|
||||
tl.store(out_ptr + row * stride_out + offs, out, mask=mask)
|
||||
|
||||
|
||||
def _hc_combine(
|
||||
def hc_combine(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
@@ -345,7 +344,7 @@ def _hc_combine_norm_kernel(
|
||||
tl.store(y_ptr + row * stride_y + offs, y, mask_inner)
|
||||
|
||||
|
||||
def _hc_combine_norm(
|
||||
def hc_combine_norm(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
@@ -392,109 +391,6 @@ def _hc_combine_norm(
|
||||
return out, y
|
||||
|
||||
|
||||
def _same_shape_fake(x: torch.Tensor, *args) -> torch.Tensor:
|
||||
return x.new_empty(x.shape)
|
||||
|
||||
|
||||
def _hc_gate_mix_fake(
|
||||
x: torch.Tensor, gate: torch.Tensor, hc_count: int
|
||||
) -> torch.Tensor:
|
||||
del gate
|
||||
return x.new_empty((x.shape[0], x.shape[1] // hc_count))
|
||||
|
||||
|
||||
def _hc_combine_fake(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
del block_output, injection_logits, hc_count
|
||||
return residual.new_empty(residual.shape)
|
||||
|
||||
|
||||
def _hc_combine_norm_fake(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
del block_output, injection_logits, norm_weight, eps, hc_count
|
||||
return residual.new_empty(residual.shape), residual.new_empty(residual.shape)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_grouped_gemma_rmsnorm",
|
||||
op_func=_grouped_gemma_rmsnorm,
|
||||
fake_impl=_same_shape_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_silu",
|
||||
op_func=_hc_silu,
|
||||
fake_impl=_same_shape_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_gate_mix",
|
||||
op_func=_hc_gate_mix,
|
||||
fake_impl=_hc_gate_mix_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_combine",
|
||||
op_func=_hc_combine,
|
||||
fake_impl=_hc_combine_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_combine_norm",
|
||||
op_func=_hc_combine_norm,
|
||||
fake_impl=_hc_combine_norm_fake,
|
||||
)
|
||||
|
||||
|
||||
def grouped_gemma_rmsnorm(
|
||||
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_grouped_gemma_rmsnorm(x, weight, eps, num_groups)
|
||||
|
||||
|
||||
def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_silu(x, hc_count)
|
||||
|
||||
|
||||
def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_gate_mix(x, gate, hc_count)
|
||||
|
||||
|
||||
def hc_combine(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_combine(
|
||||
residual, block_output, injection_logits, hc_count
|
||||
)
|
||||
|
||||
|
||||
def hc_combine_norm(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.qwen4_exp_hc_combine_norm(
|
||||
residual,
|
||||
block_output,
|
||||
injection_logits,
|
||||
norm_weight,
|
||||
eps,
|
||||
hc_count,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"grouped_gemma_rmsnorm",
|
||||
"hc_combine",
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
_TOPK_WORKSPACE_BYTES = 1024 * 1024
|
||||
_DECODE_BLOCK_N = 64
|
||||
@@ -611,7 +612,7 @@ def qsa_select_paged_prefill(
|
||||
# No row scores beyond cdiv(max_seq_len, compress_ratio) compressed
|
||||
# columns. Round up to 64 to keep the logits row stride
|
||||
# cooperative_topk-compatible.
|
||||
logits_width = triton.cdiv(triton.cdiv(max_seq_len, compress_ratio), 64) * 64
|
||||
logits_width = round_up(triton.cdiv(max_seq_len, compress_ratio), 64)
|
||||
logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1])
|
||||
|
||||
# chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB
|
||||
|
||||
@@ -38,7 +38,9 @@ def decode_opencv(
|
||||
int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
|
||||
int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
|
||||
)
|
||||
source = loader_cls._prepare_source(OpenCVVideoBackendMixin.get_video_metadata(cap))
|
||||
source = loader_cls._prepare_source(
|
||||
OpenCVVideoBackendMixin.get_video_metadata(data)
|
||||
)
|
||||
frame_idx = loader_cls.compute_frames_index_to_sample(
|
||||
source=source, target=target, **sampling_kwargs
|
||||
)
|
||||
@@ -74,10 +76,34 @@ class OpenCVVideoBackendMixin:
|
||||
raise ValueError("Could not open video stream")
|
||||
return cap
|
||||
|
||||
@classmethod
|
||||
def demux_count_frames(cls, data: bytes) -> int:
|
||||
"""Count frames with OpenCV's demuxer-only mode."""
|
||||
# NOTE: buf must stay referenced until after release().
|
||||
# OpenCV's Python stream binding does not safely own the stream object
|
||||
buf = BytesIO(data)
|
||||
demux_cap = cv2.VideoCapture(
|
||||
buf, cls.get_cv2_video_api(), [cv2.CAP_PROP_FORMAT, -1]
|
||||
)
|
||||
count = 0
|
||||
while demux_cap.grab():
|
||||
count += 1
|
||||
demux_cap.release()
|
||||
if count == 0:
|
||||
raise ValueError("Could not determine the number of video frames")
|
||||
return count
|
||||
|
||||
@staticmethod
|
||||
def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata:
|
||||
def get_video_metadata(data: bytes) -> VideoSourceMetadata:
|
||||
cap = OpenCVVideoBackendMixin.open_video_capture(data)
|
||||
total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
original_fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
if total_frames_num <= 0:
|
||||
# Streamed WebM/MKV and raw H.264 miss frame count in headers;
|
||||
# We need to use demux frames count for retrieval
|
||||
total_frames_num = OpenCVVideoBackendMixin.demux_count_frames(data)
|
||||
duration = total_frames_num / original_fps if original_fps > 0 else 0
|
||||
return VideoSourceMetadata(total_frames_num, original_fps, duration)
|
||||
# CAP_PROP_FRAME_COUNT counts every physical sample in the container,
|
||||
# overstating the presentable frames when an mp4 edit list hides the
|
||||
# decode lead-in (e.g. a lossless `ffmpeg -ss ... -c copy` trim).
|
||||
@@ -87,7 +113,7 @@ class OpenCVVideoBackendMixin:
|
||||
# may land mid-stream on corrupt files, so only trust it when the
|
||||
# stream really ends there. A one-frame tolerance keeps mere header
|
||||
# miscounts from being "corrected".
|
||||
if total_frames_num > 0 and cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
|
||||
if cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
|
||||
visible_frames = int(cap.get(cv2.CAP_PROP_POS_FRAMES))
|
||||
at_stream_end = not cap.grab()
|
||||
cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
|
||||
|
||||
@@ -89,6 +89,10 @@ CONFIG_FIELD_ALIASES: dict[str, tuple[str, ...]] = {
|
||||
`ModelArchConfigConvertorBase` reads a scalar from these; a caller that needs the
|
||||
raw value (a per-layer `list`, say) passes the aliases to `getattr_iter` itself."""
|
||||
|
||||
_ST_TRANSFORMER_MODULE_TYPES = {
|
||||
"sentence_transformers.models.Transformer",
|
||||
"sentence_transformers.base.modules.transformer.Transformer",
|
||||
}
|
||||
_ST_POOLING_MODULE_TYPES = {
|
||||
"sentence_transformers.models.Pooling",
|
||||
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
|
||||
@@ -1354,6 +1358,88 @@ def try_get_dense_modules(
|
||||
return None
|
||||
|
||||
|
||||
@cache
|
||||
def get_sentence_transformers_chat_template_kwargs(
|
||||
model: str | Path,
|
||||
revision: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Get the saved chat template settings of a Sentence Transformers
|
||||
CrossEncoder whose single Transformer module declares the `message`
|
||||
modality.
|
||||
|
||||
Sentence Transformers formats the pairs of such checkpoints as
|
||||
`query`/`document` messages with the saved chat template.
|
||||
|
||||
Args:
|
||||
model: Local checkpoint path or Hugging Face model ID.
|
||||
revision: Model revision to inspect.
|
||||
|
||||
Returns:
|
||||
The saved `processing_kwargs.chat_template` (possibly empty), or
|
||||
`None` if the checkpoint does not declare the `message` modality.
|
||||
|
||||
Raises:
|
||||
ValueError: If the saved chat template settings are unsupported.
|
||||
|
||||
"""
|
||||
try:
|
||||
st_config = get_hf_file_to_dict(
|
||||
"config_sentence_transformers.json", model, revision
|
||||
)
|
||||
if not isinstance(st_config, dict) or (
|
||||
st_config.get("model_type") != "CrossEncoder"
|
||||
):
|
||||
return None
|
||||
modules = get_hf_file_to_dict("modules.json", model, revision)
|
||||
transformer_config = get_hf_file_to_dict(
|
||||
"sentence_bert_config.json", model, revision
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to read the Sentence Transformers config", exc_info=True)
|
||||
return None
|
||||
|
||||
if isinstance(modules, dict):
|
||||
modules = modules.get("modules")
|
||||
if not (
|
||||
isinstance(modules, list)
|
||||
and len(modules) == 1
|
||||
and isinstance(modules[0], dict)
|
||||
and modules[0].get("type") in _ST_TRANSFORMER_MODULE_TYPES
|
||||
and not modules[0].get("path")
|
||||
and isinstance(transformer_config, dict)
|
||||
and transformer_config.get("transformer_task") == "sequence-classification"
|
||||
and isinstance(
|
||||
modality_config := transformer_config.get("modality_config"), dict
|
||||
)
|
||||
and "message" in modality_config
|
||||
):
|
||||
return None
|
||||
|
||||
processing_kwargs = transformer_config.get("processing_kwargs") or {}
|
||||
chat_template_kwargs = (
|
||||
processing_kwargs.get("chat_template") or {}
|
||||
if isinstance(processing_kwargs, dict)
|
||||
else None
|
||||
)
|
||||
if not isinstance(chat_template_kwargs, dict):
|
||||
raise ValueError(
|
||||
"The Sentence Transformers processing_kwargs.chat_template must be "
|
||||
"a JSON object."
|
||||
)
|
||||
chat_template_kwargs = dict(chat_template_kwargs)
|
||||
# vLLM does not restore the template suffix after truncation.
|
||||
chat_template_kwargs.pop("restore_suffix", None)
|
||||
if not isinstance(chat_template_kwargs.get("chat_template", ""), str):
|
||||
raise ValueError(
|
||||
"The Sentence Transformers chat template name must be a string."
|
||||
)
|
||||
if reserved := {"tokenize", "tools"} & chat_template_kwargs.keys():
|
||||
raise ValueError(
|
||||
f"Unsupported Sentence Transformers chat template settings: {reserved}"
|
||||
)
|
||||
return chat_template_kwargs
|
||||
|
||||
|
||||
def _read_safetensors_metadata_in_dir(local_dir: Path) -> dict[str, Any]:
|
||||
return {
|
||||
param_name: info
|
||||
|
||||
Reference in New Issue
Block a user