[Bugfix][CI] fix typos (#34934)

Signed-off-by: 1195343015 <1195343015@qq.com>
Signed-off-by: Jiayi Yan <66017932+1195343015@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Jiayi Yan
2026-03-05 17:05:46 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 8c760b6ab6
commit 6a895197fa
98 changed files with 227 additions and 366 deletions
+2 -2
View File
@@ -3106,7 +3106,7 @@ def cpu_attn_get_scheduler_metadata(
isa: str,
enable_kv_split: bool,
) -> torch.Tensor:
sheduler_metadata = torch.ops._C.get_scheduler_metadata(
scheduler_metadata = torch.ops._C.get_scheduler_metadata(
num_reqs,
num_heads,
num_kv_heads,
@@ -3119,7 +3119,7 @@ def cpu_attn_get_scheduler_metadata(
isa,
enable_kv_split,
)
return sheduler_metadata
return scheduler_metadata
def cpu_attn_reshape_and_cache(
+1 -1
View File
@@ -872,7 +872,7 @@ class CompilationConfig:
)
# Currently only eager and inductor backend are supported.
# for piecewise compilation. Custom backends are not suppported for
# for piecewise compilation. Custom backends are not supported for
# piecewise compilation. Update when more backends are supported.
if self.mode == CompilationMode.VLLM_COMPILE and self.backend not in [
"",
+1 -1
View File
@@ -59,7 +59,7 @@ class ObservabilityConfig:
enable_layerwise_nvtx_tracing: bool = False
"""Enable layerwise NVTX tracing. This traces the execution of each layer or
module in the model and attach informations such as input/output shapes to
module in the model and attach information such as input/output shapes to
nvtx range markers. Noted that this doesn't work with CUDA graphs enabled."""
enable_mfu_metrics: bool = False
+1 -1
View File
@@ -592,7 +592,7 @@ class VllmConfig:
If the user configuration does not specify a value for a default field
and if the default field is still None after all user selections are
applied, then default values will be applied to the field. User speciied
applied, then default values will be applied to the field. User specified
fields will not be overridden by the default.
Args:
+1 -1
View File
@@ -44,7 +44,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
rank_in_pack = np.zeros_like(pack_index, dtype=np.int64)
return pack_index, rank_in_pack
# Sort and get indices in decending order
# Sort and get indices in descending order
indices = np.argsort(-weight, axis=-1)
pack_index = np.full((num_layers, num_groups), -1, dtype=np.int64)
@@ -129,7 +129,7 @@ class KVConnectorRole(enum.Enum):
class KVConnectorHandshakeMetadata(ABC): # noqa: B024
"""
Metadata used for out of band connector handshake between
P/D workers. This needs to serializeable.
P/D workers. This needs to serializable.
"""
pass
@@ -398,7 +398,7 @@ class ReqMeta:
)
def need_gpu_interm_buffer(lmcache_config: LMCacheEngineConfig):
def need_gpu_interim_buffer(lmcache_config: LMCacheEngineConfig):
return not lmcache_config.enable_pd
@@ -497,7 +497,7 @@ def _init_lmcache_engine(
use_mla,
)
use_gpu = need_gpu_interm_buffer(lmcache_config)
use_gpu = need_gpu_interim_buffer(lmcache_config)
vllm_gpu_connector: (
VLLMBufferLayerwiseGPUConnector
| VLLMPagedMemGPUConnectorV2
@@ -481,7 +481,7 @@ class MooncakeConnectorWorker:
)
self._remote_agents: dict[EngineId, dict[int, dict[int, str]]] = {}
self._pending_bootstrap_querys: dict[str, asyncio.Event] = {}
self._pending_bootstrap_queries: dict[str, asyncio.Event] = {}
self.side_channel_port: int = 0 # we will bind it in register_kv_caches()
self.engine_id: EngineId = engine_id
self.tp_rank = get_tensor_model_parallel_rank()
@@ -1077,7 +1077,7 @@ class MooncakeConnectorWorker:
response = self._xfer_resp_decoder.decode(ret_msg)
if response.status == MooncakeXferResponseStatus.ERROR:
logger.error(
"Error happens during tranfering kvcache for %s: %s",
"Error happens during transferring kvcache for %s: %s",
req_ids,
response.err_msg,
)
@@ -1140,8 +1140,8 @@ class MooncakeConnectorWorker:
)
# Always notify others regardless of connection success or failure.
self._pending_bootstrap_querys[remote_bootstrap_addr].set()
del self._pending_bootstrap_querys[remote_bootstrap_addr]
self._pending_bootstrap_queries[remote_bootstrap_addr].set()
del self._pending_bootstrap_queries[remote_bootstrap_addr]
def receive_kv(
self,
@@ -1171,11 +1171,11 @@ class MooncakeConnectorWorker:
pull_metas: dict[ReqId, PullReqMeta],
):
remote_bootstrap_addr = next(iter(pull_metas.values())).remote_bootstrap_addr
if remote_bootstrap_addr not in self._pending_bootstrap_querys:
self._pending_bootstrap_querys[remote_bootstrap_addr] = asyncio.Event()
if remote_bootstrap_addr not in self._pending_bootstrap_queries:
self._pending_bootstrap_queries[remote_bootstrap_addr] = asyncio.Event()
await self._connect_to_prefiller_bootstrap(remote_bootstrap_addr)
else:
await self._pending_bootstrap_querys[remote_bootstrap_addr].wait()
await self._pending_bootstrap_queries[remote_bootstrap_addr].wait()
if remote_engine_id not in self._remote_agents:
logger.error(
@@ -720,7 +720,7 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
# (engine_idx, transfer_tupe) -> (metric with bounded labels)
# (engine_idx, transfer_type) -> (metric with bounded labels)
self.histogram_transfer_size: dict[tuple[int, str], PromMetricT] = {}
self.counter_kv_bytes: dict[tuple[int, str], PromMetricT] = {}
self.counter_kv_transfer_time: dict[tuple[int, str], PromMetricT] = {}
+3 -3
View File
@@ -1647,9 +1647,9 @@ class OpenAIServingResponses(OpenAIServing):
# TODO: in streaming, we noticed this bug:
# https://github.com/vllm-project/vllm/issues/25697
await self._initialize_tool_sessions(request, context, exit_stack)
processer = self._process_harmony_streaming_events
processor = self._process_harmony_streaming_events
else:
processer = self._process_simple_streaming_events
processor = self._process_simple_streaming_events
# TODO Hanchen make sampling params to include the structural tag
initial_response = ResponsesResponse.from_request(
@@ -1677,7 +1677,7 @@ class OpenAIServingResponses(OpenAIServing):
)
try:
async for event_data in processer(
async for event_data in processor(
request,
sampling_params,
result_generator,
+1 -1
View File
@@ -1520,7 +1520,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
os.getenv("VLLM_DEEPEP_BUFFER_SIZE_MB", "1024")
),
# Force DeepEP to use intranode kernel for inter-node communication in
# high throughput mode. This is useful archive higher prefill throuhgput
# high throughput mode. This is useful archive higher prefill throughput
# on system supports multi-node nvlink (e.g GB200).
"VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE": lambda: bool(
int(os.getenv("VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE", "0"))
+1 -1
View File
@@ -175,7 +175,7 @@ class DPMetadata:
# Get the cumulative tokens across sequence parallel ranks.
# In this case the input to the MoEs will be distributed w.r.t both
# DP and TP rank.
# When sp_size==1, this is just the cummulative num tokens across DP.
# When sp_size==1, this is just the cumulative num tokens across DP.
def cu_tokens_across_sp(self, sp_size: int) -> torch.Tensor:
num_tokens_across_sp_cpu = (
self.num_tokens_across_dp_cpu - 1 + sp_size
+2 -2
View File
@@ -57,10 +57,10 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
input_parallel = input_
else:
# TODO: simplify code below
splitted_input = split_tensor_along_last_dim(
split_input = split_tensor_along_last_dim(
input_, num_partitions=self.tp_size
)
input_parallel = splitted_input[self.tp_rank].contiguous()
input_parallel = split_input[self.tp_rank].contiguous()
# Matrix multiply.
bias_ = (
+3 -3
View File
@@ -11,7 +11,7 @@ from vllm.lora.lora_weights import LoRALayerWeights
from vllm.lora.peft_helper import PEFTHelper
from vllm.lora.utils import (
get_lora_id,
is_base_embeddding_weights,
is_base_embedding_weights,
parse_fine_tuned_lora_name,
)
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
@@ -86,7 +86,7 @@ class LoRAModel:
pin_memory = str(device) == "cpu" and is_pin_memory_available()
loras: dict[str, LoRALayerWeights] = {}
for tensor_name, tensor in tensors.items():
if is_base_embeddding_weights(tensor_name):
if is_base_embedding_weights(tensor_name):
continue
# Skip modules based on model-defined prefixes (e.g., MTP layers)
if skip_prefixes and cls._should_skip_module(tensor_name, skip_prefixes):
@@ -162,7 +162,7 @@ class LoRAModel:
def check_unexpected_modules(modules: dict):
for lora_module in modules.keys(): # noqa
if is_base_embeddding_weights(lora_module):
if is_base_embedding_weights(lora_module):
continue
# Handle PEFT file format where experts.base_layer is the
# gate_up_proj and experts is the down_proj
+1 -1
View File
@@ -193,7 +193,7 @@ def parse_fine_tuned_lora_name(
raise ValueError(f"{name} is unsupported LoRA weight")
def is_base_embeddding_weights(name: str) -> bool:
def is_base_embedding_weights(name: str) -> bool:
# hardcoded subfixes for input & output embedding weights
embedding_suffixes = (
".embed_tokens.base_layer.weight",
@@ -82,7 +82,7 @@ class CPUWNA16LinearKernel(MPLinearKernel):
weight = weight.permute(0, 2, 1).reshape(input_size, output_size).contiguous()
weight = pack_quantized_values_into_int32(weight, self.config.weight_type, 1)
# make 16 output channel as a block and transpose to the make
# the block contigous
# the block contiguous
weight = (
weight.view(input_size, -1, 16 // pack_factor)
.permute(1, 0, 2)
@@ -2540,7 +2540,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
)
# workspace
# |------- N tokens --------|--------- N*dcp_size tokens ----------|
# |<- use for loca_gather ->|<--------- use for allgather -------->|
# |<- use for local_gather ->|<--------- use for allgather -------->|
allgather_offset = workspace.shape[0] // (dcp_world_size + 1)
assert allgather_offset * (dcp_world_size + 1) == workspace.shape[0]
assert toks <= allgather_offset
@@ -394,5 +394,5 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
# No support for LoRA in flashinfer_cutlass_fused_moe.
# See TODOs in flashinfer functions runMoe and runMoeMinLantency.
# See TODOs in flashinfer functions runMoe and runMoeMinLatency.
raise NotImplementedError("LoRA is not supported for flashinfer_cutlass_moe")
@@ -409,7 +409,7 @@ def batched_fused_marlin_moe(
Note that the moe_align_block_size function indicates,
- What rows of the A matrix (hidden_states) to access during the
matmul, via sorted_ids output.
- What expert_id to use for each block matmul, via expert_ids ouptut.
- What expert_id to use for each block matmul, via expert_ids output.
In the batched version, the tokens are already grouped/batched by experts
they subscribe to. Due to this, we can represent the batched hidden_states
@@ -606,7 +606,7 @@ class FusedMoEExperts(ABC):
"""
Whether the kernel supports deployment in particular parallel config.
Can be overriden if a kernel does not support EP, SP or some other
Can be overridden if a kernel does not support EP, SP or some other
configuration.
"""
raise NotImplementedError
@@ -620,7 +620,7 @@ class FusedMoEExperts(ABC):
"""
Whether the kernel supports a routing method (e.g. GroupedTopK).
Can be overriden by monolithic kernels that execute the router
Can be overridden by monolithic kernels that execute the router
in addition to the experts if certain routers are not supported.
"""
return True
@@ -633,7 +633,7 @@ class FusedMoEExperts(ABC):
"""
Whether a kernel supports a particular dtype for router logits input.
Can be overriden by monolithic kernels that execute the router
Can be overridden by monolithic kernels that execute the router
in addition to the experts if certain dtypes are not supported.
"""
return True
+2 -2
View File
@@ -1502,10 +1502,10 @@ class RowParallelLinear(LinearBase):
if self.input_is_parallel:
input_parallel = input_
else:
splitted_input = split_tensor_along_last_dim(
split_input = split_tensor_along_last_dim(
input_, num_partitions=self.tp_size
)
input_parallel = splitted_input[self.tp_rank].contiguous()
input_parallel = split_input[self.tp_rank].contiguous()
# Matrix multiply.
assert self.quant_method is not None
+1 -1
View File
@@ -35,7 +35,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
"""Pluggable MLA layer which allows OOT backends to add
custom implementations of the outer MLA layer (including rope & o_proj).
Note that currently oot platforms can still use CustomOp.register_oot to
replace MLA layer entirly, although we use PluggableLayer to register
replace MLA layer entirely, although we use PluggableLayer to register
this layer now.
This class takes positions and hidden_states as input.
@@ -191,7 +191,7 @@ class CompressedTensorsConfig(QuantizationConfig):
"""
Helper function to update target_scheme_map
since linear layers get fused into FusedMoE
targetting 'Linear' needs to also match
targeting 'Linear' needs to also match
FusedMoE modules.
"""
if (
@@ -2445,7 +2445,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
w2_scale=layer.w2_weight_scale, # group scale
g1_alphas=layer.w13_weight_chan_scale,
g2_alphas=layer.w2_weight_chan_scale,
per_act_token_quant=True, # always use dynamc per-token
per_act_token_quant=True, # always use dynamic per-token
per_out_ch_quant=True, # always use per-channel
)
@@ -261,7 +261,7 @@ class CPUAWQLinearMethod(LinearMethodBase):
zeros = pack_cols(zeros, bits, group_num, output_size).contiguous()
# make 16 output channel as a block and transpose to
# the make the block contigous
# the make the block contiguous
weight = pack_cols(weight, bits, input_size, output_size)
weight = (
weight.view(input_size, -1, 16 // pack_factor)
@@ -199,7 +199,7 @@ class TorchAOConfig(QuantizationConfig):
@classmethod
def from_config_dict_json(cls, config_dict_json: str) -> "TorchAOConfig":
"""Iniitalize class from a config_dict json string, got from
"""Initialize class from a config_dict json string, got from
torchao_config_object = some AOBaseConfig object
json.dumps(config_to_dict(torchao_config_object))
"""
@@ -255,7 +255,7 @@ def _flashinfer_fp8_blockscale_gemm_impl(
This batch-size-dependent selection is essential for maintaining model accuracy.
Benchmarks on GSM8K show a significant accuracy gap (88% vs 95%) for DeepSeek-V3.1
when using FlashInfer's DeepGEMM on M>=32. The M < 32 strategy fixes the accurracy
when using FlashInfer's DeepGEMM on M>=32. The M < 32 strategy fixes the accuracy
drop.
Args:
@@ -39,7 +39,7 @@ def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
def check_machete_supports_shape(
in_features: int, out_featrues: int
in_features: int, out_features: int
) -> tuple[bool, str | None]:
if in_features % MACHETE_PREPACKED_BLOCK_SHAPE[0] != 0:
return (
@@ -47,7 +47,7 @@ def check_machete_supports_shape(
"Input features size must be divisible by "
f"{MACHETE_PREPACKED_BLOCK_SHAPE[0]}",
)
if out_featrues % MACHETE_PREPACKED_BLOCK_SHAPE[1] != 0:
if out_features % MACHETE_PREPACKED_BLOCK_SHAPE[1] != 0:
return (
False,
"Output features size must be divisible by "
@@ -237,7 +237,7 @@ class ApplyRotaryEmb(CustomOp):
Arguments of apply_rotary_emb() in vllm_flash_attn:
x: [batch_size, seq_len, nheads, headdim]
cos, sin: [seqlen_rotary, rotary_dim / 2]
interleaved: defalut as False (Neox-style).
interleaved: default as False (Neox-style).
...
"""
interleaved = not self.is_neox_style
@@ -259,7 +259,7 @@ class ApplyRotaryEmb(CustomOp):
Arguments of apply_rotary() in flash_attn:
x: [batch_size, seq_len, nheads, headdim]
cos, sin: [seqlen_rotary, rotary_dim / 2]
interleaved: defalut as False (Neox-style).
interleaved: default as False (Neox-style).
...
"""
interleaved = not self.is_neox_style
+6 -6
View File
@@ -342,7 +342,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
visual_token_mask = visual_token_mask.repeat(1, self.hidden_size).bool()
text_token_mask = ~visual_token_mask
final_experts_hidden_states = torch.zeros_like(hidden_states)
final_shared_ouput = (
final_shared_output = (
torch.zeros_like(hidden_states) if self.has_shared_experts else None
)
@@ -356,26 +356,26 @@ class Ernie4_5_VLMoeMoE(nn.Module):
text_router_logits, _ = self.text_experts_gate(
text_hidden_states.to(dtype=torch.float32)
)
text_shared_ouput, text_experts_output = self.text_experts(
text_shared_output, text_experts_output = self.text_experts(
hidden_states=text_hidden_states, router_logits=text_router_logits
)
final_experts_hidden_states[text_token_mask] = text_experts_output.flatten()
if self.has_shared_experts:
final_shared_ouput[text_token_mask] = text_shared_ouput.flatten()
final_shared_output[text_token_mask] = text_shared_output.flatten()
vision_router_logits, _ = self.vision_experts_gate(
vision_hidden_states.to(dtype=torch.float32)
)
vision_shared_ouput, vision_experts_output = self.vision_experts(
vision_shared_output, vision_experts_output = self.vision_experts(
hidden_states=vision_hidden_states, router_logits=vision_router_logits
)
final_experts_hidden_states[visual_token_mask] = (
vision_experts_output.flatten()
)
if self.has_shared_experts:
final_shared_ouput[visual_token_mask] = vision_shared_ouput.flatten()
final_shared_output[visual_token_mask] = vision_shared_output.flatten()
final_hidden_states = (final_shared_ouput, final_experts_hidden_states)
final_hidden_states = (final_shared_output, final_experts_hidden_states)
else:
# only text modal input
text_router_logits, _ = self.text_experts_gate(
+1 -1
View File
@@ -107,7 +107,7 @@ class Conv2dSubsampling(nn.Module):
)
self.subsampling = 4
left_context = right_context = 3 # both exclude currect frame
left_context = right_context = 3 # both exclude current frame
self.context = left_context + 1 + right_context # 7
def forward(
+9 -9
View File
@@ -115,7 +115,7 @@ class EncoderLayerSANM(nn.Module):
hidden_states: torch.Tensor,
mask: torch.Tensor | None = None,
cache=None,
mask_shfit_chunk=None,
mask_shift_chunk=None,
mask_att_chunk_encoder=None,
):
residual = hidden_states
@@ -125,14 +125,14 @@ class EncoderLayerSANM(nn.Module):
hidden_states = residual + self.self_attn(
hidden_states,
mask,
mask_shfit_chunk=mask_shfit_chunk,
mask_shift_chunk=mask_shift_chunk,
mask_att_chunk_encoder=mask_att_chunk_encoder,
)
else:
hidden_states = self.self_attn(
hidden_states,
mask,
mask_shfit_chunk=mask_shfit_chunk,
mask_shift_chunk=mask_shift_chunk,
mask_att_chunk_encoder=mask_att_chunk_encoder,
)
@@ -140,7 +140,7 @@ class EncoderLayerSANM(nn.Module):
hidden_states = self.norm2(hidden_states)
hidden_states = residual + self.feed_forward(hidden_states)
return hidden_states, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder
return hidden_states, mask, cache, mask_shift_chunk, mask_att_chunk_encoder
class MultiHeadedAttentionSANM(nn.Module):
@@ -183,13 +183,13 @@ class MultiHeadedAttentionSANM(nn.Module):
self,
inputs: torch.Tensor,
mask: torch.Tensor,
mask_shfit_chunk: torch.Tensor = None,
mask_shift_chunk: torch.Tensor = None,
):
b, t, d = inputs.size()
if mask is not None:
mask = torch.reshape(mask, (b, -1, 1))
if mask_shfit_chunk is not None:
mask = mask * mask_shfit_chunk
if mask_shift_chunk is not None:
mask = mask * mask_shift_chunk
inputs = inputs * mask
x = inputs.transpose(1, 2)
@@ -243,11 +243,11 @@ class MultiHeadedAttentionSANM(nn.Module):
self,
hidden_states: torch.Tensor,
mask: torch.Tensor,
mask_shfit_chunk: torch.Tensor = None,
mask_shift_chunk: torch.Tensor = None,
mask_att_chunk_encoder: torch.Tensor = None,
):
q_h, k_h, v_h, v = self.forward_qkv(hidden_states)
fsmn_memory = self.forward_fsmn(v, mask, mask_shfit_chunk)
fsmn_memory = self.forward_fsmn(v, mask, mask_shift_chunk)
q_h = q_h * self.d_k ** (-0.5)
scores = torch.matmul(q_h, k_h.transpose(-2, -1))
att_outs = self.forward_attention(v_h, scores, mask, mask_att_chunk_encoder)
+1 -1
View File
@@ -646,7 +646,7 @@ class IsaacImageProcessor:
return_tensors: str | TensorType | None,
**kwargs: Unpack[IsaacImageProcessorKwargs],
) -> BatchFeature:
"""Preprocess images into format compatibile with vLLM input processing."""
"""Preprocess images into format compatible with vLLM input processing."""
all_pixel_values: list[torch.Tensor] = []
all_image_grids: list[torch.Tensor] = []
+1 -1
View File
@@ -299,7 +299,7 @@ class KeyeVisionEmbeddings(nn.Module):
)
(
batch_size,
squence_len,
sequence_len,
channel,
height,
width,
+8 -8
View File
@@ -238,7 +238,7 @@ class LongcatRouter(nn.Module):
self,
config: FlashConfig,
zero_expert_num: int,
rounter_params_dtype: torch.dtype,
router_params_dtype: torch.dtype,
prefix: str = "",
):
super().__init__()
@@ -252,12 +252,12 @@ class LongcatRouter(nn.Module):
config.hidden_size,
self.n_routed_experts,
bias=config.router_bias,
params_dtype=rounter_params_dtype,
params_dtype=router_params_dtype,
quant_config=None,
prefix=f"{prefix}.classifier",
)
self.e_score_correction_bias = nn.Parameter(
torch.zeros((self.n_routed_experts), dtype=rounter_params_dtype)
torch.zeros((self.n_routed_experts), dtype=router_params_dtype)
)
def forward(self, hidden_states):
@@ -281,14 +281,14 @@ class LongcatMoe(nn.Module):
super().__init__()
self.hidden_size = hidden_size
# Gate always runs at half / full precision for now.
self.rounter_params_dtype = params_dtype
self.router_params_dtype = params_dtype
if config.router_dtype == "float32":
self.rounter_params_dtype = torch.float32
self.router_params_dtype = torch.float32
self.router = LongcatRouter(
config=config,
zero_expert_num=config.zero_expert_num,
rounter_params_dtype=self.rounter_params_dtype,
router_params_dtype=self.router_params_dtype,
prefix=f"{prefix}.gate",
)
@@ -309,7 +309,7 @@ class LongcatMoe(nn.Module):
prefix=f"{prefix}.experts",
enable_eplb=enable_eplb,
routed_scaling_factor=config.routed_scaling_factor,
router_logits_dtype=self.rounter_params_dtype,
router_logits_dtype=self.router_params_dtype,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -329,7 +329,7 @@ class LongcatMoe(nn.Module):
hidden_states_padded = hidden_states
router_logits_full = self.router(
hidden_states_padded.to(self.rounter_params_dtype)
hidden_states_padded.to(self.router_params_dtype)
)
# ZeroExpertFusedMoE handles routing memoization and zero expert computation
+4 -4
View File
@@ -1321,14 +1321,14 @@ def get_image_size(image: ImageInput) -> ImageSize:
raise ValueError(f"Unknown image type: {type(image)}")
def exif_tranpose(
def exif_transpose(
images: ImageInput | None,
) -> ImageInput | None:
if images is None:
return None
if images is not None and isinstance(images, (list, tuple)):
images = [
exif_tranpose(img) if isinstance(img, Image) else img for img in images
exif_transpose(img) if isinstance(img, Image) else img for img in images
]
elif images is not None and isinstance(images, Image):
images = ImageOps.exif_transpose(images)
@@ -1667,7 +1667,7 @@ class Molmo2ProcessorWrapper:
**kwargs: object,
) -> BatchFeature:
inputs = [text]
images = exif_tranpose(images)
images = exif_transpose(images)
if getattr(self.processor, "image_processor", None) is not None:
inputs.append(images)
if getattr(self.processor, "video_processor", None) is not None:
@@ -2352,7 +2352,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def get_image_replacement_molmo2(item_idx: int) -> list[int]:
images = mm_items.get_items("image", ImageProcessorItems)
image = images.get(item_idx)
image = exif_tranpose(image)
image = exif_transpose(image)
resize_nrows, resize_cols = processor.get_base_grid_size(is_video=False)
if use_single_crop_col_tokens is not None:
+2 -2
View File
@@ -349,7 +349,7 @@ class NemotronHMoEDecoderLayer(nn.Module):
super().__init__()
self.config = config
# Get per-layer config for heterogeneous models if exsist
# Get per-layer config for heterogeneous models if exists
get_layer_config = getattr(config, "get_nemotron_h_config_for_layer", None)
layer_config = get_layer_config(layer_idx) if get_layer_config else config
@@ -517,7 +517,7 @@ class NemotronHAttentionDecoderLayer(nn.Module):
) -> None:
super().__init__()
# Get per-layer config for heterogeneous models if exsist
# Get per-layer config for heterogeneous models if exists
get_layer_config = getattr(config, "get_nemotron_h_config_for_layer", None)
layer_config = get_layer_config(layer_idx) if get_layer_config else config
+1 -1
View File
@@ -486,7 +486,7 @@ class SiglipVisionEmbeddings(nn.Module):
)
(
batch_size,
squence_len,
sequence_len,
channel,
height,
width,
+6 -6
View File
@@ -689,19 +689,19 @@ class ConformerEncoder(TransformerEncoderBase):
default False.
ext_pw_out_channel: int, optional
the number of channel for CNN
before depthwise_seperable_CNN.
before depthwise_separable_CNN.
If 0 then use linear. default 0.
ext_pw_kernel_size: int, optional
kernel size of N before depthwise_seperable_CNN.
kernel size of N before depthwise_separable_CNN.
only work for ext_pw_out_channel > 0.
default 1
depthwise_seperable_out_channel: int, optional
the number of channel for
depthwise_seperable_CNN.
depthwise_separable_CNN.
default 256.
depthwise_multiplier: int, optional
the number of multiplier for
depthwise_seperable_CNN.
depthwise_separable_CNN.
default 1.
chunk_se: int, optional
0 for offline SE.
@@ -711,7 +711,7 @@ class ConformerEncoder(TransformerEncoderBase):
by only the current chunk.
default 0.
kernel_size: int, optional
the number of kernels for depthwise_seperable_CNN.
the number of kernels for depthwise_separable_CNN.
default 3.
activation: str, optional
FeedForward block activation.
@@ -721,7 +721,7 @@ class ConformerEncoder(TransformerEncoderBase):
activation function used in ConvModule part
of the conformer, default "relu".
conv_glu_type: str, optional
activation used use glu in depthwise_seperable_CNN,
activation used use glu in depthwise_separable_CNN,
default "sigmoid"
bias_in_glu: bool, optional
if set to True, use additive bias in the weight module
+3 -3
View File
@@ -217,8 +217,8 @@ class GLUPointWiseConv(nn.Module):
return x
class DepthWiseSeperableConv1d(nn.Module):
"""DepthWiseSeperableConv1d module used in Convnet module
class DepthWiseSeparableConv1d(nn.Module):
"""DepthWiseSeparableConv1d module used in ConvNet module
for the conformer, for more details see:
https://arxiv.org/pdf/2005.08100v1.pdf
@@ -390,7 +390,7 @@ class ConvModule(nn.Module):
else:
padding = (kernel_size - 1) // 2
self.dw_sep_conv_1d = DepthWiseSeperableConv1d(
self.dw_sep_conv_1d = DepthWiseSeparableConv1d(
input_dim,
depthwise_seperable_out_channel,
kernel_size,
+1 -1
View File
@@ -916,7 +916,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
self, max_pixels: int | None = None
) -> ImageSize:
# NOTE: Simply processing a huge size with _get_vision_info might not give a
# size that maximizes the number of featrues, i.e., the number of (merged)
# size that maximizes the number of features, i.e., the number of (merged)
# patches. This is because the number of patches limits the allowed aspect
# ratios. For example, suppose the maximum number of patches is 1280. A square
# image cannot be broken down into 1280 patches, so feeding a giant square image
+2 -2
View File
@@ -459,14 +459,14 @@ class Step3VLProcessor:
image_inputs = {}
text_inputs = self.tokenizer(text)
else:
splitted_images_data = self._split_images(images)
split_images_data = self._split_images(images)
pixel_values_lst = []
patch_pixel_values_lst = []
patch_newline_mask_lst = []
image_repl_str_lst = []
image_repl_ids_lst = []
num_patches = []
for raw_img, img_patches, patch_newline_mask in splitted_images_data:
for raw_img, img_patches, patch_newline_mask in split_images_data:
pixel_values_lst.extend(self._convert_images_to_pixel_values([raw_img]))
if len(img_patches) > 0:
+1 -1
View File
@@ -353,7 +353,7 @@ class FusedMoEBlock(nn.Module):
if swiglu_limit not in (None, 0):
swiglu_limit = float(swiglu_limit)
assert swiglu_limit == 7.0, (
"Swiglu limit in fused moe block only suport 7.0 now."
"Swiglu limit in fused moe block only support 7.0 now."
)
activation = "swiglustep"
logger.debug(
+3 -3
View File
@@ -18,7 +18,7 @@ logger = init_logger(__name__)
class Ernie45ReasoningParser(BaseThinkingReasoningParser):
"""
Reasoning parser for Ernie45 thinking model.
The Ernie45 thinking model ouput format is
The Ernie45 thinking model output format is
abc\n</think>\n\n<response>\ndef\n</response>\n
or abc\n</think>\ndef
"""
@@ -73,7 +73,7 @@ class Ernie45ReasoningParser(BaseThinkingReasoningParser):
Extract reasoning content from a delta message.
Handles streaming output where previous + delta = current.
Uses token IDs for faster processing.
The Ernie45 thinking model ouput format is
The Ernie45 thinking model output format is
abc\n</think>\n\n<response>\ndef\n</response>\n
or abc\n</think>\ndef
- 'abc' goes to reasoning
@@ -148,7 +148,7 @@ class Ernie45ReasoningParser(BaseThinkingReasoningParser):
) -> tuple[str | None, str | None]:
"""
Extract reasoning content from the model output.
The Ernie45 thinking model ouput format is
The Ernie45 thinking model output format is
abc\n</think>\n\n\n<response>\ndef\n</response>\n
or abc\n</think>\ndef
- 'abc' goes to reasoning
+1 -1
View File
@@ -564,7 +564,7 @@ def replace_vision_chunk_video_placeholder(
mm_data: "MultiModalDataDict",
video_placeholder: str | None,
) -> str | list[int]:
# get video placehoder, replace it with runtime video-chunk prompts
# get video placeholder, replace it with runtime video-chunk prompts
if video_placeholder and isinstance(prompt_raw, str):
video_prompts = build_video_prompts_from_mm_data(mm_data)
+1 -1
View File
@@ -1,5 +1,5 @@
"""
Schemas and utilites for preprocessing inputs.
Schemas and utilities for preprocessing inputs.
"""
# SPDX-License-Identifier: Apache-2.0
+1 -1
View File
@@ -1,5 +1,5 @@
"""
Schemas and utilites for tokenization inputs.
Schemas and utilities for tokenization inputs.
"""
# SPDX-License-Identifier: Apache-2.0
+2 -2
View File
@@ -169,7 +169,7 @@ def _prepare_apply_chat_template_tools_and_messages(
tool.pop(tool_key)
logger.warning_once(
f"'{tool_key}' is not supported by mistral-common for tools. "
"It has been poped from the tool definition."
"It has been popped from the tool definition."
)
if tool["type"] == "function":
function_keys = list(tool["function"].keys())
@@ -178,7 +178,7 @@ def _prepare_apply_chat_template_tools_and_messages(
tool["function"].pop(function_key)
logger.warning_once(
f"'{function_key}' is not supported by mistral-common "
"for function tools. It has been poped from the "
"for function tools. It has been popped from the "
"function definition."
)
else:
@@ -402,7 +402,7 @@ class Ovis2_5Processor(ProcessorMixin):
images = [images]
elif video is not None:
is_video = True
# type of vidoe in dummy_mm_data is np.ndarray
# type of video in dummy_mm_data is np.ndarray
if isinstance(video, np.ndarray):
images = []
for i in range(video.shape[0]):
+2 -2
View File
@@ -174,7 +174,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
query_start_loc = query_start_loc[: num_decodes + 1]
block_table_tensor = block_table_tensor[:num_decodes]
sheduler_metadata = ops.cpu_attn_get_scheduler_metadata(
scheduler_metadata = ops.cpu_attn_get_scheduler_metadata(
num_reqs=num_reqs,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
@@ -197,7 +197,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
seq_lens=seq_lens,
block_table=block_table_tensor,
slot_mapping=slot_mapping,
scheduler_metadata=sheduler_metadata,
scheduler_metadata=scheduler_metadata,
causal=causal,
use_sdpa_prefill=self.use_sdpa_prefill,
num_decode_tokens=num_decode_tokens,
+1 -1
View File
@@ -383,7 +383,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
# Return a tensor of shape (#requests, #max blocks)
state_indices_tensor = common_attn_metadata.block_table_tensor
# Additional cache-related varaiables:
# Additional cache-related variables:
mamba_block_size = self.kv_cache_spec.block_size
(
block_idx_last_computed_token,
@@ -49,14 +49,14 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
# For FP8 sparse attention we have two impelementations:
# For FP8 sparse attention we have two implementations:
# 1. Mixed batch mode: use the FP8 decode kernel for both prefill and decode this is
# done by treating all tokens as single batch.
# 2. Separate prefill and decode mode: use the BF16 prefill kernel for prefill
# (upconverting the FP8 cache to BF16 then calling the prefill kernel) and using
# the FP8 decode kernel for decode.
# Currently we use #1 when the number of heads per rank is low (i.e. TP) since the BF16
# prefill kernel requires padding the numer of heads to 128 while the decode does not
# prefill kernel requires padding the number of heads to 128 while the decode does not
# so when the per ranke head count is below MIN_HEADS_FOR_BF16_PREFILL we use the mixed
# batch mode (#2).
MIN_HEADS_FOR_BF16_PREFILL = 32
@@ -126,7 +126,7 @@ class FlashMLASparseBackend(AttentionBackend):
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
if cache_dtype_str == "fp8_ds_mla":
# custom storage fromat is 656 bytes
# custom storage format is 656 bytes
# see FlashMLA readme.md for details
return (num_blocks, block_size, 656)
else:
+3 -3
View File
@@ -370,7 +370,7 @@ class AiterFlashAttentionMetadata:
slot_mapping: torch.Tensor
block_table: torch.Tensor
# prefill and deocde split
# prefill and decode split
num_decodes: int
num_decode_tokens: int
num_prefills: int
@@ -1099,7 +1099,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
extend_tokens_slice = slice(
num_decode_tokens, num_decode_tokens + num_extend_tokens
)
extend_querys = query[extend_tokens_slice]
extend_queries = query[extend_tokens_slice]
extend_keys = key[extend_tokens_slice]
extend_values = value[extend_tokens_slice]
extend_outputs = output[extend_tokens_slice]
@@ -1110,7 +1110,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
v_scale = attn_metadata.v_scale
self.extend_forward(
attn_metadata=attn_metadata,
query=extend_querys,
query=extend_queries,
key=extend_keys,
value=extend_values,
key_cache=key_cache,
+1 -1
View File
@@ -863,7 +863,7 @@ class MambaManager(SingleTypeKVCacheManager):
):
# Mamba can't rely on blocks generated by other requests in the current step
# To put it in the next step, we return num_gpu_blocks + 1 so
# that kv_cache_manager will think there is no enough blocks to allocte now
# that kv_cache_manager will think there is no enough blocks to allocate now
# and don't schedule it in the current step.
return self.block_pool.num_gpu_blocks + 1
if self.mamba_cache_mode != "align":
+2 -2
View File
@@ -1724,11 +1724,11 @@ class DPEngineCoreProc(EngineCoreProc):
"""
Send notifications to EngineCoreClient, which can then forward
the notifications to other engine core processes. It is used for:
1) In scale up: new core engines to notify exisiting core engines
1) In scale up: new core engines to notify existing core engines
that they are ready;
2) In scale down: removing core engines to notify EngineCoreClient
so EngineCoreClient can release their ray placement groups;
3) Both scale up/down: to notify EngineCoreClient that exisiting
3) Both scale up/down: to notify EngineCoreClient that existing
core engines have already switched to the new parallel setup.
"""
if vllm_config is None:
+1 -1
View File
@@ -194,7 +194,7 @@ class InputProcessor:
@staticmethod
def assign_request_id(request: EngineCoreRequest):
"""Replace the externally supplied request ID with an internal request ID
that adds 8 random characters in order to ensure uniquness.
that adds 8 random characters in order to ensure uniqueness.
"""
if request.external_req_id is not None:
raise ValueError(
+1 -1
View File
@@ -197,7 +197,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
transfer = self._transfers.popleft()
transfer_time = (
transfer.start_event.elapsed_time(transfer.end_event) * 1e-3
) # elapsed_time is in miliseconds
) # elapsed_time is in milliseconds
result = TransferResult(
job_id=transfer.job_id,
success=True,
+11 -11
View File
@@ -905,7 +905,7 @@ class GPUModelRunner(
Args:
scheduler_output: The scheduler output.
"""
# Attention free models have zero kv_cache_goups, however models
# Attention free models have zero kv_cache_groups, however models
# like Mamba are also attention free but use the kv_cache for
# keeping its internal state. This is why we check the number
# of kv_cache groups instead of solely checking
@@ -1065,7 +1065,7 @@ class GPUModelRunner(
# of the request. for example:
# fist step: num_computed_tokens = 0, spec_tokens = [],
# prev_num_draft_len = 0.
# second step: num_computed_tokens = 100(prompt lenth),
# second step: num_computed_tokens = 100(prompt length),
# spec_tokens = [a,b], prev_num_draft_len = 0.
# third step: num_computed_tokens = 100 + 2, spec_tokens = [c,d],
# prev_num_draft_len = 2.
@@ -1412,30 +1412,30 @@ class GPUModelRunner(
prev_draft_token_indices.extend(range(start, start + draft_len))
indices_match &= prev_index == flattened_index
max_flattened_index = max(max_flattened_index, flattened_index)
num_commmon_tokens = len(sample_flattened_indices)
num_common_tokens = len(sample_flattened_indices)
total_without_spec = total_num_scheduled_tokens - total_num_spec_tokens
if num_commmon_tokens < total_without_spec:
if num_common_tokens < total_without_spec:
# If not all requests are decodes from the last iteration,
# We need to copy the input_ids_cpu to the GPU first.
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
if self.enable_prompt_embeds:
self.inputs_embeds.copy_to_gpu(total_num_scheduled_tokens)
self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
if num_commmon_tokens == 0:
if num_common_tokens == 0:
# No requests in common with the previous iteration
# So input_ids.cpu will have all the input ids.
return
if indices_match and max_flattened_index == (num_commmon_tokens - 1):
if indices_match and max_flattened_index == (num_common_tokens - 1):
# Common-case optimization: the batch is unchanged
# and no reordering happened.
# The indices are both the same permutation of 0..N-1 so
# we can copy directly using a single slice.
self.input_ids.gpu[:num_commmon_tokens].copy_(
self.input_batch.prev_sampled_token_ids[:num_commmon_tokens, 0],
self.input_ids.gpu[:num_common_tokens].copy_(
self.input_batch.prev_sampled_token_ids[:num_common_tokens, 0],
non_blocking=True,
)
if self.enable_prompt_embeds:
self.is_token_ids.gpu[:num_commmon_tokens] = True
self.is_token_ids.gpu[:num_common_tokens] = True
return
# Upload the index tensors asynchronously so the scatter can be non-blocking.
sampled_tokens_index_tensor = torch.tensor(
@@ -4383,7 +4383,7 @@ class GPUModelRunner(
self.model.compile(fullgraph=True, backend=backend)
return
# for other compilation modes, cudagraph behavior is controlled by
# CudagraphWraper and CudagraphDispatcher of vllm.
# CudagraphWrapper and CudagraphDispatcher of vllm.
# wrap the model with full cudagraph wrapper if needed.
cudagraph_mode = self.compilation_config.cudagraph_mode
@@ -4444,7 +4444,7 @@ class GPUModelRunner(
:param weights_path: path to load weights from if weights_iterator is not
provided. Use path of original model if neither is provided.
:param is_checkpoint_format: set to False if weights have already been processed
into kernel format (repacking, renaming, ect.)
into kernel format (repacking, renaming, etc.)
"""
# TODO(@kylesayrs): generalize to all runners and loaders
# argument validation