[Bugfix][CI] fix typos (#34934)
Signed-off-by: 1195343015 <1195343015@qq.com> Signed-off-by: Jiayi Yan <66017932+1195343015@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
8c760b6ab6
commit
6a895197fa
+2
-2
@@ -3106,7 +3106,7 @@ def cpu_attn_get_scheduler_metadata(
|
||||
isa: str,
|
||||
enable_kv_split: bool,
|
||||
) -> torch.Tensor:
|
||||
sheduler_metadata = torch.ops._C.get_scheduler_metadata(
|
||||
scheduler_metadata = torch.ops._C.get_scheduler_metadata(
|
||||
num_reqs,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
@@ -3119,7 +3119,7 @@ def cpu_attn_get_scheduler_metadata(
|
||||
isa,
|
||||
enable_kv_split,
|
||||
)
|
||||
return sheduler_metadata
|
||||
return scheduler_metadata
|
||||
|
||||
|
||||
def cpu_attn_reshape_and_cache(
|
||||
|
||||
@@ -872,7 +872,7 @@ class CompilationConfig:
|
||||
)
|
||||
|
||||
# Currently only eager and inductor backend are supported.
|
||||
# for piecewise compilation. Custom backends are not suppported for
|
||||
# for piecewise compilation. Custom backends are not supported for
|
||||
# piecewise compilation. Update when more backends are supported.
|
||||
if self.mode == CompilationMode.VLLM_COMPILE and self.backend not in [
|
||||
"",
|
||||
|
||||
@@ -59,7 +59,7 @@ class ObservabilityConfig:
|
||||
|
||||
enable_layerwise_nvtx_tracing: bool = False
|
||||
"""Enable layerwise NVTX tracing. This traces the execution of each layer or
|
||||
module in the model and attach informations such as input/output shapes to
|
||||
module in the model and attach information such as input/output shapes to
|
||||
nvtx range markers. Noted that this doesn't work with CUDA graphs enabled."""
|
||||
|
||||
enable_mfu_metrics: bool = False
|
||||
|
||||
+1
-1
@@ -592,7 +592,7 @@ class VllmConfig:
|
||||
|
||||
If the user configuration does not specify a value for a default field
|
||||
and if the default field is still None after all user selections are
|
||||
applied, then default values will be applied to the field. User speciied
|
||||
applied, then default values will be applied to the field. User specified
|
||||
fields will not be overridden by the default.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -44,7 +44,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
rank_in_pack = np.zeros_like(pack_index, dtype=np.int64)
|
||||
return pack_index, rank_in_pack
|
||||
|
||||
# Sort and get indices in decending order
|
||||
# Sort and get indices in descending order
|
||||
indices = np.argsort(-weight, axis=-1)
|
||||
|
||||
pack_index = np.full((num_layers, num_groups), -1, dtype=np.int64)
|
||||
|
||||
@@ -129,7 +129,7 @@ class KVConnectorRole(enum.Enum):
|
||||
class KVConnectorHandshakeMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Metadata used for out of band connector handshake between
|
||||
P/D workers. This needs to serializeable.
|
||||
P/D workers. This needs to serializable.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
@@ -398,7 +398,7 @@ class ReqMeta:
|
||||
)
|
||||
|
||||
|
||||
def need_gpu_interm_buffer(lmcache_config: LMCacheEngineConfig):
|
||||
def need_gpu_interim_buffer(lmcache_config: LMCacheEngineConfig):
|
||||
return not lmcache_config.enable_pd
|
||||
|
||||
|
||||
@@ -497,7 +497,7 @@ def _init_lmcache_engine(
|
||||
use_mla,
|
||||
)
|
||||
|
||||
use_gpu = need_gpu_interm_buffer(lmcache_config)
|
||||
use_gpu = need_gpu_interim_buffer(lmcache_config)
|
||||
vllm_gpu_connector: (
|
||||
VLLMBufferLayerwiseGPUConnector
|
||||
| VLLMPagedMemGPUConnectorV2
|
||||
|
||||
@@ -481,7 +481,7 @@ class MooncakeConnectorWorker:
|
||||
)
|
||||
|
||||
self._remote_agents: dict[EngineId, dict[int, dict[int, str]]] = {}
|
||||
self._pending_bootstrap_querys: dict[str, asyncio.Event] = {}
|
||||
self._pending_bootstrap_queries: dict[str, asyncio.Event] = {}
|
||||
self.side_channel_port: int = 0 # we will bind it in register_kv_caches()
|
||||
self.engine_id: EngineId = engine_id
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
@@ -1077,7 +1077,7 @@ class MooncakeConnectorWorker:
|
||||
response = self._xfer_resp_decoder.decode(ret_msg)
|
||||
if response.status == MooncakeXferResponseStatus.ERROR:
|
||||
logger.error(
|
||||
"Error happens during tranfering kvcache for %s: %s",
|
||||
"Error happens during transferring kvcache for %s: %s",
|
||||
req_ids,
|
||||
response.err_msg,
|
||||
)
|
||||
@@ -1140,8 +1140,8 @@ class MooncakeConnectorWorker:
|
||||
)
|
||||
|
||||
# Always notify others regardless of connection success or failure.
|
||||
self._pending_bootstrap_querys[remote_bootstrap_addr].set()
|
||||
del self._pending_bootstrap_querys[remote_bootstrap_addr]
|
||||
self._pending_bootstrap_queries[remote_bootstrap_addr].set()
|
||||
del self._pending_bootstrap_queries[remote_bootstrap_addr]
|
||||
|
||||
def receive_kv(
|
||||
self,
|
||||
@@ -1171,11 +1171,11 @@ class MooncakeConnectorWorker:
|
||||
pull_metas: dict[ReqId, PullReqMeta],
|
||||
):
|
||||
remote_bootstrap_addr = next(iter(pull_metas.values())).remote_bootstrap_addr
|
||||
if remote_bootstrap_addr not in self._pending_bootstrap_querys:
|
||||
self._pending_bootstrap_querys[remote_bootstrap_addr] = asyncio.Event()
|
||||
if remote_bootstrap_addr not in self._pending_bootstrap_queries:
|
||||
self._pending_bootstrap_queries[remote_bootstrap_addr] = asyncio.Event()
|
||||
await self._connect_to_prefiller_bootstrap(remote_bootstrap_addr)
|
||||
else:
|
||||
await self._pending_bootstrap_querys[remote_bootstrap_addr].wait()
|
||||
await self._pending_bootstrap_queries[remote_bootstrap_addr].wait()
|
||||
|
||||
if remote_engine_id not in self._remote_agents:
|
||||
logger.error(
|
||||
|
||||
@@ -720,7 +720,7 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
# (engine_idx, transfer_tupe) -> (metric with bounded labels)
|
||||
# (engine_idx, transfer_type) -> (metric with bounded labels)
|
||||
self.histogram_transfer_size: dict[tuple[int, str], PromMetricT] = {}
|
||||
self.counter_kv_bytes: dict[tuple[int, str], PromMetricT] = {}
|
||||
self.counter_kv_transfer_time: dict[tuple[int, str], PromMetricT] = {}
|
||||
|
||||
@@ -1647,9 +1647,9 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
# TODO: in streaming, we noticed this bug:
|
||||
# https://github.com/vllm-project/vllm/issues/25697
|
||||
await self._initialize_tool_sessions(request, context, exit_stack)
|
||||
processer = self._process_harmony_streaming_events
|
||||
processor = self._process_harmony_streaming_events
|
||||
else:
|
||||
processer = self._process_simple_streaming_events
|
||||
processor = self._process_simple_streaming_events
|
||||
# TODO Hanchen make sampling params to include the structural tag
|
||||
|
||||
initial_response = ResponsesResponse.from_request(
|
||||
@@ -1677,7 +1677,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
)
|
||||
|
||||
try:
|
||||
async for event_data in processer(
|
||||
async for event_data in processor(
|
||||
request,
|
||||
sampling_params,
|
||||
result_generator,
|
||||
|
||||
+1
-1
@@ -1520,7 +1520,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
os.getenv("VLLM_DEEPEP_BUFFER_SIZE_MB", "1024")
|
||||
),
|
||||
# Force DeepEP to use intranode kernel for inter-node communication in
|
||||
# high throughput mode. This is useful archive higher prefill throuhgput
|
||||
# high throughput mode. This is useful archive higher prefill throughput
|
||||
# on system supports multi-node nvlink (e.g GB200).
|
||||
"VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE": lambda: bool(
|
||||
int(os.getenv("VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE", "0"))
|
||||
|
||||
@@ -175,7 +175,7 @@ class DPMetadata:
|
||||
# Get the cumulative tokens across sequence parallel ranks.
|
||||
# In this case the input to the MoEs will be distributed w.r.t both
|
||||
# DP and TP rank.
|
||||
# When sp_size==1, this is just the cummulative num tokens across DP.
|
||||
# When sp_size==1, this is just the cumulative num tokens across DP.
|
||||
def cu_tokens_across_sp(self, sp_size: int) -> torch.Tensor:
|
||||
num_tokens_across_sp_cpu = (
|
||||
self.num_tokens_across_dp_cpu - 1 + sp_size
|
||||
|
||||
@@ -57,10 +57,10 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
input_parallel = input_
|
||||
else:
|
||||
# TODO: simplify code below
|
||||
splitted_input = split_tensor_along_last_dim(
|
||||
split_input = split_tensor_along_last_dim(
|
||||
input_, num_partitions=self.tp_size
|
||||
)
|
||||
input_parallel = splitted_input[self.tp_rank].contiguous()
|
||||
input_parallel = split_input[self.tp_rank].contiguous()
|
||||
|
||||
# Matrix multiply.
|
||||
bias_ = (
|
||||
|
||||
@@ -11,7 +11,7 @@ from vllm.lora.lora_weights import LoRALayerWeights
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
from vllm.lora.utils import (
|
||||
get_lora_id,
|
||||
is_base_embeddding_weights,
|
||||
is_base_embedding_weights,
|
||||
parse_fine_tuned_lora_name,
|
||||
)
|
||||
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
|
||||
@@ -86,7 +86,7 @@ class LoRAModel:
|
||||
pin_memory = str(device) == "cpu" and is_pin_memory_available()
|
||||
loras: dict[str, LoRALayerWeights] = {}
|
||||
for tensor_name, tensor in tensors.items():
|
||||
if is_base_embeddding_weights(tensor_name):
|
||||
if is_base_embedding_weights(tensor_name):
|
||||
continue
|
||||
# Skip modules based on model-defined prefixes (e.g., MTP layers)
|
||||
if skip_prefixes and cls._should_skip_module(tensor_name, skip_prefixes):
|
||||
@@ -162,7 +162,7 @@ class LoRAModel:
|
||||
|
||||
def check_unexpected_modules(modules: dict):
|
||||
for lora_module in modules.keys(): # noqa
|
||||
if is_base_embeddding_weights(lora_module):
|
||||
if is_base_embedding_weights(lora_module):
|
||||
continue
|
||||
# Handle PEFT file format where experts.base_layer is the
|
||||
# gate_up_proj and experts is the down_proj
|
||||
|
||||
+1
-1
@@ -193,7 +193,7 @@ def parse_fine_tuned_lora_name(
|
||||
raise ValueError(f"{name} is unsupported LoRA weight")
|
||||
|
||||
|
||||
def is_base_embeddding_weights(name: str) -> bool:
|
||||
def is_base_embedding_weights(name: str) -> bool:
|
||||
# hardcoded subfixes for input & output embedding weights
|
||||
embedding_suffixes = (
|
||||
".embed_tokens.base_layer.weight",
|
||||
|
||||
@@ -82,7 +82,7 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
weight = weight.permute(0, 2, 1).reshape(input_size, output_size).contiguous()
|
||||
weight = pack_quantized_values_into_int32(weight, self.config.weight_type, 1)
|
||||
# make 16 output channel as a block and transpose to the make
|
||||
# the block contigous
|
||||
# the block contiguous
|
||||
weight = (
|
||||
weight.view(input_size, -1, 16 // pack_factor)
|
||||
.permute(1, 0, 2)
|
||||
|
||||
@@ -2540,7 +2540,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
)
|
||||
# workspace
|
||||
# |------- N tokens --------|--------- N*dcp_size tokens ----------|
|
||||
# |<- use for loca_gather ->|<--------- use for allgather -------->|
|
||||
# |<- use for local_gather ->|<--------- use for allgather -------->|
|
||||
allgather_offset = workspace.shape[0] // (dcp_world_size + 1)
|
||||
assert allgather_offset * (dcp_world_size + 1) == workspace.shape[0]
|
||||
assert toks <= allgather_offset
|
||||
|
||||
@@ -394,5 +394,5 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
|
||||
# No support for LoRA in flashinfer_cutlass_fused_moe.
|
||||
# See TODOs in flashinfer functions runMoe and runMoeMinLantency.
|
||||
# See TODOs in flashinfer functions runMoe and runMoeMinLatency.
|
||||
raise NotImplementedError("LoRA is not supported for flashinfer_cutlass_moe")
|
||||
|
||||
@@ -409,7 +409,7 @@ def batched_fused_marlin_moe(
|
||||
Note that the moe_align_block_size function indicates,
|
||||
- What rows of the A matrix (hidden_states) to access during the
|
||||
matmul, via sorted_ids output.
|
||||
- What expert_id to use for each block matmul, via expert_ids ouptut.
|
||||
- What expert_id to use for each block matmul, via expert_ids output.
|
||||
|
||||
In the batched version, the tokens are already grouped/batched by experts
|
||||
they subscribe to. Due to this, we can represent the batched hidden_states
|
||||
|
||||
@@ -606,7 +606,7 @@ class FusedMoEExperts(ABC):
|
||||
"""
|
||||
Whether the kernel supports deployment in particular parallel config.
|
||||
|
||||
Can be overriden if a kernel does not support EP, SP or some other
|
||||
Can be overridden if a kernel does not support EP, SP or some other
|
||||
configuration.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -620,7 +620,7 @@ class FusedMoEExperts(ABC):
|
||||
"""
|
||||
Whether the kernel supports a routing method (e.g. GroupedTopK).
|
||||
|
||||
Can be overriden by monolithic kernels that execute the router
|
||||
Can be overridden by monolithic kernels that execute the router
|
||||
in addition to the experts if certain routers are not supported.
|
||||
"""
|
||||
return True
|
||||
@@ -633,7 +633,7 @@ class FusedMoEExperts(ABC):
|
||||
"""
|
||||
Whether a kernel supports a particular dtype for router logits input.
|
||||
|
||||
Can be overriden by monolithic kernels that execute the router
|
||||
Can be overridden by monolithic kernels that execute the router
|
||||
in addition to the experts if certain dtypes are not supported.
|
||||
"""
|
||||
return True
|
||||
|
||||
@@ -1502,10 +1502,10 @@ class RowParallelLinear(LinearBase):
|
||||
if self.input_is_parallel:
|
||||
input_parallel = input_
|
||||
else:
|
||||
splitted_input = split_tensor_along_last_dim(
|
||||
split_input = split_tensor_along_last_dim(
|
||||
input_, num_partitions=self.tp_size
|
||||
)
|
||||
input_parallel = splitted_input[self.tp_rank].contiguous()
|
||||
input_parallel = split_input[self.tp_rank].contiguous()
|
||||
|
||||
# Matrix multiply.
|
||||
assert self.quant_method is not None
|
||||
|
||||
@@ -35,7 +35,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
"""Pluggable MLA layer which allows OOT backends to add
|
||||
custom implementations of the outer MLA layer (including rope & o_proj).
|
||||
Note that currently oot platforms can still use CustomOp.register_oot to
|
||||
replace MLA layer entirly, although we use PluggableLayer to register
|
||||
replace MLA layer entirely, although we use PluggableLayer to register
|
||||
this layer now.
|
||||
|
||||
This class takes positions and hidden_states as input.
|
||||
|
||||
@@ -191,7 +191,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
"""
|
||||
Helper function to update target_scheme_map
|
||||
since linear layers get fused into FusedMoE
|
||||
targetting 'Linear' needs to also match
|
||||
targeting 'Linear' needs to also match
|
||||
FusedMoE modules.
|
||||
"""
|
||||
if (
|
||||
|
||||
+1
-1
@@ -2445,7 +2445,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_scale=layer.w2_weight_scale, # group scale
|
||||
g1_alphas=layer.w13_weight_chan_scale,
|
||||
g2_alphas=layer.w2_weight_chan_scale,
|
||||
per_act_token_quant=True, # always use dynamc per-token
|
||||
per_act_token_quant=True, # always use dynamic per-token
|
||||
per_out_ch_quant=True, # always use per-channel
|
||||
)
|
||||
|
||||
|
||||
@@ -261,7 +261,7 @@ class CPUAWQLinearMethod(LinearMethodBase):
|
||||
|
||||
zeros = pack_cols(zeros, bits, group_num, output_size).contiguous()
|
||||
# make 16 output channel as a block and transpose to
|
||||
# the make the block contigous
|
||||
# the make the block contiguous
|
||||
weight = pack_cols(weight, bits, input_size, output_size)
|
||||
weight = (
|
||||
weight.view(input_size, -1, 16 // pack_factor)
|
||||
|
||||
@@ -199,7 +199,7 @@ class TorchAOConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def from_config_dict_json(cls, config_dict_json: str) -> "TorchAOConfig":
|
||||
"""Iniitalize class from a config_dict json string, got from
|
||||
"""Initialize class from a config_dict json string, got from
|
||||
torchao_config_object = some AOBaseConfig object
|
||||
json.dumps(config_to_dict(torchao_config_object))
|
||||
"""
|
||||
|
||||
@@ -255,7 +255,7 @@ def _flashinfer_fp8_blockscale_gemm_impl(
|
||||
|
||||
This batch-size-dependent selection is essential for maintaining model accuracy.
|
||||
Benchmarks on GSM8K show a significant accuracy gap (88% vs 95%) for DeepSeek-V3.1
|
||||
when using FlashInfer's DeepGEMM on M>=32. The M < 32 strategy fixes the accurracy
|
||||
when using FlashInfer's DeepGEMM on M>=32. The M < 32 strategy fixes the accuracy
|
||||
drop.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -39,7 +39,7 @@ def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
|
||||
|
||||
|
||||
def check_machete_supports_shape(
|
||||
in_features: int, out_featrues: int
|
||||
in_features: int, out_features: int
|
||||
) -> tuple[bool, str | None]:
|
||||
if in_features % MACHETE_PREPACKED_BLOCK_SHAPE[0] != 0:
|
||||
return (
|
||||
@@ -47,7 +47,7 @@ def check_machete_supports_shape(
|
||||
"Input features size must be divisible by "
|
||||
f"{MACHETE_PREPACKED_BLOCK_SHAPE[0]}",
|
||||
)
|
||||
if out_featrues % MACHETE_PREPACKED_BLOCK_SHAPE[1] != 0:
|
||||
if out_features % MACHETE_PREPACKED_BLOCK_SHAPE[1] != 0:
|
||||
return (
|
||||
False,
|
||||
"Output features size must be divisible by "
|
||||
|
||||
@@ -237,7 +237,7 @@ class ApplyRotaryEmb(CustomOp):
|
||||
Arguments of apply_rotary_emb() in vllm_flash_attn:
|
||||
x: [batch_size, seq_len, nheads, headdim]
|
||||
cos, sin: [seqlen_rotary, rotary_dim / 2]
|
||||
interleaved: defalut as False (Neox-style).
|
||||
interleaved: default as False (Neox-style).
|
||||
...
|
||||
"""
|
||||
interleaved = not self.is_neox_style
|
||||
@@ -259,7 +259,7 @@ class ApplyRotaryEmb(CustomOp):
|
||||
Arguments of apply_rotary() in flash_attn:
|
||||
x: [batch_size, seq_len, nheads, headdim]
|
||||
cos, sin: [seqlen_rotary, rotary_dim / 2]
|
||||
interleaved: defalut as False (Neox-style).
|
||||
interleaved: default as False (Neox-style).
|
||||
...
|
||||
"""
|
||||
interleaved = not self.is_neox_style
|
||||
|
||||
@@ -342,7 +342,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
|
||||
visual_token_mask = visual_token_mask.repeat(1, self.hidden_size).bool()
|
||||
text_token_mask = ~visual_token_mask
|
||||
final_experts_hidden_states = torch.zeros_like(hidden_states)
|
||||
final_shared_ouput = (
|
||||
final_shared_output = (
|
||||
torch.zeros_like(hidden_states) if self.has_shared_experts else None
|
||||
)
|
||||
|
||||
@@ -356,26 +356,26 @@ class Ernie4_5_VLMoeMoE(nn.Module):
|
||||
text_router_logits, _ = self.text_experts_gate(
|
||||
text_hidden_states.to(dtype=torch.float32)
|
||||
)
|
||||
text_shared_ouput, text_experts_output = self.text_experts(
|
||||
text_shared_output, text_experts_output = self.text_experts(
|
||||
hidden_states=text_hidden_states, router_logits=text_router_logits
|
||||
)
|
||||
final_experts_hidden_states[text_token_mask] = text_experts_output.flatten()
|
||||
if self.has_shared_experts:
|
||||
final_shared_ouput[text_token_mask] = text_shared_ouput.flatten()
|
||||
final_shared_output[text_token_mask] = text_shared_output.flatten()
|
||||
|
||||
vision_router_logits, _ = self.vision_experts_gate(
|
||||
vision_hidden_states.to(dtype=torch.float32)
|
||||
)
|
||||
vision_shared_ouput, vision_experts_output = self.vision_experts(
|
||||
vision_shared_output, vision_experts_output = self.vision_experts(
|
||||
hidden_states=vision_hidden_states, router_logits=vision_router_logits
|
||||
)
|
||||
final_experts_hidden_states[visual_token_mask] = (
|
||||
vision_experts_output.flatten()
|
||||
)
|
||||
if self.has_shared_experts:
|
||||
final_shared_ouput[visual_token_mask] = vision_shared_ouput.flatten()
|
||||
final_shared_output[visual_token_mask] = vision_shared_output.flatten()
|
||||
|
||||
final_hidden_states = (final_shared_ouput, final_experts_hidden_states)
|
||||
final_hidden_states = (final_shared_output, final_experts_hidden_states)
|
||||
else:
|
||||
# only text modal input
|
||||
text_router_logits, _ = self.text_experts_gate(
|
||||
|
||||
@@ -107,7 +107,7 @@ class Conv2dSubsampling(nn.Module):
|
||||
)
|
||||
|
||||
self.subsampling = 4
|
||||
left_context = right_context = 3 # both exclude currect frame
|
||||
left_context = right_context = 3 # both exclude current frame
|
||||
self.context = left_context + 1 + right_context # 7
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -115,7 +115,7 @@ class EncoderLayerSANM(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
mask: torch.Tensor | None = None,
|
||||
cache=None,
|
||||
mask_shfit_chunk=None,
|
||||
mask_shift_chunk=None,
|
||||
mask_att_chunk_encoder=None,
|
||||
):
|
||||
residual = hidden_states
|
||||
@@ -125,14 +125,14 @@ class EncoderLayerSANM(nn.Module):
|
||||
hidden_states = residual + self.self_attn(
|
||||
hidden_states,
|
||||
mask,
|
||||
mask_shfit_chunk=mask_shfit_chunk,
|
||||
mask_shift_chunk=mask_shift_chunk,
|
||||
mask_att_chunk_encoder=mask_att_chunk_encoder,
|
||||
)
|
||||
else:
|
||||
hidden_states = self.self_attn(
|
||||
hidden_states,
|
||||
mask,
|
||||
mask_shfit_chunk=mask_shfit_chunk,
|
||||
mask_shift_chunk=mask_shift_chunk,
|
||||
mask_att_chunk_encoder=mask_att_chunk_encoder,
|
||||
)
|
||||
|
||||
@@ -140,7 +140,7 @@ class EncoderLayerSANM(nn.Module):
|
||||
hidden_states = self.norm2(hidden_states)
|
||||
hidden_states = residual + self.feed_forward(hidden_states)
|
||||
|
||||
return hidden_states, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder
|
||||
return hidden_states, mask, cache, mask_shift_chunk, mask_att_chunk_encoder
|
||||
|
||||
|
||||
class MultiHeadedAttentionSANM(nn.Module):
|
||||
@@ -183,13 +183,13 @@ class MultiHeadedAttentionSANM(nn.Module):
|
||||
self,
|
||||
inputs: torch.Tensor,
|
||||
mask: torch.Tensor,
|
||||
mask_shfit_chunk: torch.Tensor = None,
|
||||
mask_shift_chunk: torch.Tensor = None,
|
||||
):
|
||||
b, t, d = inputs.size()
|
||||
if mask is not None:
|
||||
mask = torch.reshape(mask, (b, -1, 1))
|
||||
if mask_shfit_chunk is not None:
|
||||
mask = mask * mask_shfit_chunk
|
||||
if mask_shift_chunk is not None:
|
||||
mask = mask * mask_shift_chunk
|
||||
inputs = inputs * mask
|
||||
|
||||
x = inputs.transpose(1, 2)
|
||||
@@ -243,11 +243,11 @@ class MultiHeadedAttentionSANM(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
mask: torch.Tensor,
|
||||
mask_shfit_chunk: torch.Tensor = None,
|
||||
mask_shift_chunk: torch.Tensor = None,
|
||||
mask_att_chunk_encoder: torch.Tensor = None,
|
||||
):
|
||||
q_h, k_h, v_h, v = self.forward_qkv(hidden_states)
|
||||
fsmn_memory = self.forward_fsmn(v, mask, mask_shfit_chunk)
|
||||
fsmn_memory = self.forward_fsmn(v, mask, mask_shift_chunk)
|
||||
q_h = q_h * self.d_k ** (-0.5)
|
||||
scores = torch.matmul(q_h, k_h.transpose(-2, -1))
|
||||
att_outs = self.forward_attention(v_h, scores, mask, mask_att_chunk_encoder)
|
||||
|
||||
@@ -646,7 +646,7 @@ class IsaacImageProcessor:
|
||||
return_tensors: str | TensorType | None,
|
||||
**kwargs: Unpack[IsaacImageProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
"""Preprocess images into format compatibile with vLLM input processing."""
|
||||
"""Preprocess images into format compatible with vLLM input processing."""
|
||||
|
||||
all_pixel_values: list[torch.Tensor] = []
|
||||
all_image_grids: list[torch.Tensor] = []
|
||||
|
||||
@@ -299,7 +299,7 @@ class KeyeVisionEmbeddings(nn.Module):
|
||||
)
|
||||
(
|
||||
batch_size,
|
||||
squence_len,
|
||||
sequence_len,
|
||||
channel,
|
||||
height,
|
||||
width,
|
||||
|
||||
@@ -238,7 +238,7 @@ class LongcatRouter(nn.Module):
|
||||
self,
|
||||
config: FlashConfig,
|
||||
zero_expert_num: int,
|
||||
rounter_params_dtype: torch.dtype,
|
||||
router_params_dtype: torch.dtype,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
@@ -252,12 +252,12 @@ class LongcatRouter(nn.Module):
|
||||
config.hidden_size,
|
||||
self.n_routed_experts,
|
||||
bias=config.router_bias,
|
||||
params_dtype=rounter_params_dtype,
|
||||
params_dtype=router_params_dtype,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.classifier",
|
||||
)
|
||||
self.e_score_correction_bias = nn.Parameter(
|
||||
torch.zeros((self.n_routed_experts), dtype=rounter_params_dtype)
|
||||
torch.zeros((self.n_routed_experts), dtype=router_params_dtype)
|
||||
)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
@@ -281,14 +281,14 @@ class LongcatMoe(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
# Gate always runs at half / full precision for now.
|
||||
self.rounter_params_dtype = params_dtype
|
||||
self.router_params_dtype = params_dtype
|
||||
if config.router_dtype == "float32":
|
||||
self.rounter_params_dtype = torch.float32
|
||||
self.router_params_dtype = torch.float32
|
||||
|
||||
self.router = LongcatRouter(
|
||||
config=config,
|
||||
zero_expert_num=config.zero_expert_num,
|
||||
rounter_params_dtype=self.rounter_params_dtype,
|
||||
router_params_dtype=self.router_params_dtype,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
@@ -309,7 +309,7 @@ class LongcatMoe(nn.Module):
|
||||
prefix=f"{prefix}.experts",
|
||||
enable_eplb=enable_eplb,
|
||||
routed_scaling_factor=config.routed_scaling_factor,
|
||||
router_logits_dtype=self.rounter_params_dtype,
|
||||
router_logits_dtype=self.router_params_dtype,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -329,7 +329,7 @@ class LongcatMoe(nn.Module):
|
||||
hidden_states_padded = hidden_states
|
||||
|
||||
router_logits_full = self.router(
|
||||
hidden_states_padded.to(self.rounter_params_dtype)
|
||||
hidden_states_padded.to(self.router_params_dtype)
|
||||
)
|
||||
|
||||
# ZeroExpertFusedMoE handles routing memoization and zero expert computation
|
||||
|
||||
@@ -1321,14 +1321,14 @@ def get_image_size(image: ImageInput) -> ImageSize:
|
||||
raise ValueError(f"Unknown image type: {type(image)}")
|
||||
|
||||
|
||||
def exif_tranpose(
|
||||
def exif_transpose(
|
||||
images: ImageInput | None,
|
||||
) -> ImageInput | None:
|
||||
if images is None:
|
||||
return None
|
||||
if images is not None and isinstance(images, (list, tuple)):
|
||||
images = [
|
||||
exif_tranpose(img) if isinstance(img, Image) else img for img in images
|
||||
exif_transpose(img) if isinstance(img, Image) else img for img in images
|
||||
]
|
||||
elif images is not None and isinstance(images, Image):
|
||||
images = ImageOps.exif_transpose(images)
|
||||
@@ -1667,7 +1667,7 @@ class Molmo2ProcessorWrapper:
|
||||
**kwargs: object,
|
||||
) -> BatchFeature:
|
||||
inputs = [text]
|
||||
images = exif_tranpose(images)
|
||||
images = exif_transpose(images)
|
||||
if getattr(self.processor, "image_processor", None) is not None:
|
||||
inputs.append(images)
|
||||
if getattr(self.processor, "video_processor", None) is not None:
|
||||
@@ -2352,7 +2352,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
def get_image_replacement_molmo2(item_idx: int) -> list[int]:
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image = images.get(item_idx)
|
||||
image = exif_tranpose(image)
|
||||
image = exif_transpose(image)
|
||||
|
||||
resize_nrows, resize_cols = processor.get_base_grid_size(is_video=False)
|
||||
if use_single_crop_col_tokens is not None:
|
||||
|
||||
@@ -349,7 +349,7 @@ class NemotronHMoEDecoderLayer(nn.Module):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
|
||||
# Get per-layer config for heterogeneous models if exsist
|
||||
# Get per-layer config for heterogeneous models if exists
|
||||
get_layer_config = getattr(config, "get_nemotron_h_config_for_layer", None)
|
||||
layer_config = get_layer_config(layer_idx) if get_layer_config else config
|
||||
|
||||
@@ -517,7 +517,7 @@ class NemotronHAttentionDecoderLayer(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
# Get per-layer config for heterogeneous models if exsist
|
||||
# Get per-layer config for heterogeneous models if exists
|
||||
get_layer_config = getattr(config, "get_nemotron_h_config_for_layer", None)
|
||||
layer_config = get_layer_config(layer_idx) if get_layer_config else config
|
||||
|
||||
|
||||
@@ -486,7 +486,7 @@ class SiglipVisionEmbeddings(nn.Module):
|
||||
)
|
||||
(
|
||||
batch_size,
|
||||
squence_len,
|
||||
sequence_len,
|
||||
channel,
|
||||
height,
|
||||
width,
|
||||
|
||||
@@ -689,19 +689,19 @@ class ConformerEncoder(TransformerEncoderBase):
|
||||
default False.
|
||||
ext_pw_out_channel: int, optional
|
||||
the number of channel for CNN
|
||||
before depthwise_seperable_CNN.
|
||||
before depthwise_separable_CNN.
|
||||
If 0 then use linear. default 0.
|
||||
ext_pw_kernel_size: int, optional
|
||||
kernel size of N before depthwise_seperable_CNN.
|
||||
kernel size of N before depthwise_separable_CNN.
|
||||
only work for ext_pw_out_channel > 0.
|
||||
default 1
|
||||
depthwise_seperable_out_channel: int, optional
|
||||
the number of channel for
|
||||
depthwise_seperable_CNN.
|
||||
depthwise_separable_CNN.
|
||||
default 256.
|
||||
depthwise_multiplier: int, optional
|
||||
the number of multiplier for
|
||||
depthwise_seperable_CNN.
|
||||
depthwise_separable_CNN.
|
||||
default 1.
|
||||
chunk_se: int, optional
|
||||
0 for offline SE.
|
||||
@@ -711,7 +711,7 @@ class ConformerEncoder(TransformerEncoderBase):
|
||||
by only the current chunk.
|
||||
default 0.
|
||||
kernel_size: int, optional
|
||||
the number of kernels for depthwise_seperable_CNN.
|
||||
the number of kernels for depthwise_separable_CNN.
|
||||
default 3.
|
||||
activation: str, optional
|
||||
FeedForward block activation.
|
||||
@@ -721,7 +721,7 @@ class ConformerEncoder(TransformerEncoderBase):
|
||||
activation function used in ConvModule part
|
||||
of the conformer, default "relu".
|
||||
conv_glu_type: str, optional
|
||||
activation used use glu in depthwise_seperable_CNN,
|
||||
activation used use glu in depthwise_separable_CNN,
|
||||
default "sigmoid"
|
||||
bias_in_glu: bool, optional
|
||||
if set to True, use additive bias in the weight module
|
||||
|
||||
@@ -217,8 +217,8 @@ class GLUPointWiseConv(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
class DepthWiseSeperableConv1d(nn.Module):
|
||||
"""DepthWiseSeperableConv1d module used in Convnet module
|
||||
class DepthWiseSeparableConv1d(nn.Module):
|
||||
"""DepthWiseSeparableConv1d module used in ConvNet module
|
||||
for the conformer, for more details see:
|
||||
https://arxiv.org/pdf/2005.08100v1.pdf
|
||||
|
||||
@@ -390,7 +390,7 @@ class ConvModule(nn.Module):
|
||||
else:
|
||||
padding = (kernel_size - 1) // 2
|
||||
|
||||
self.dw_sep_conv_1d = DepthWiseSeperableConv1d(
|
||||
self.dw_sep_conv_1d = DepthWiseSeparableConv1d(
|
||||
input_dim,
|
||||
depthwise_seperable_out_channel,
|
||||
kernel_size,
|
||||
|
||||
@@ -916,7 +916,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
self, max_pixels: int | None = None
|
||||
) -> ImageSize:
|
||||
# NOTE: Simply processing a huge size with _get_vision_info might not give a
|
||||
# size that maximizes the number of featrues, i.e., the number of (merged)
|
||||
# size that maximizes the number of features, i.e., the number of (merged)
|
||||
# patches. This is because the number of patches limits the allowed aspect
|
||||
# ratios. For example, suppose the maximum number of patches is 1280. A square
|
||||
# image cannot be broken down into 1280 patches, so feeding a giant square image
|
||||
|
||||
@@ -459,14 +459,14 @@ class Step3VLProcessor:
|
||||
image_inputs = {}
|
||||
text_inputs = self.tokenizer(text)
|
||||
else:
|
||||
splitted_images_data = self._split_images(images)
|
||||
split_images_data = self._split_images(images)
|
||||
pixel_values_lst = []
|
||||
patch_pixel_values_lst = []
|
||||
patch_newline_mask_lst = []
|
||||
image_repl_str_lst = []
|
||||
image_repl_ids_lst = []
|
||||
num_patches = []
|
||||
for raw_img, img_patches, patch_newline_mask in splitted_images_data:
|
||||
for raw_img, img_patches, patch_newline_mask in split_images_data:
|
||||
pixel_values_lst.extend(self._convert_images_to_pixel_values([raw_img]))
|
||||
|
||||
if len(img_patches) > 0:
|
||||
|
||||
@@ -353,7 +353,7 @@ class FusedMoEBlock(nn.Module):
|
||||
if swiglu_limit not in (None, 0):
|
||||
swiglu_limit = float(swiglu_limit)
|
||||
assert swiglu_limit == 7.0, (
|
||||
"Swiglu limit in fused moe block only suport 7.0 now."
|
||||
"Swiglu limit in fused moe block only support 7.0 now."
|
||||
)
|
||||
activation = "swiglustep"
|
||||
logger.debug(
|
||||
|
||||
@@ -18,7 +18,7 @@ logger = init_logger(__name__)
|
||||
class Ernie45ReasoningParser(BaseThinkingReasoningParser):
|
||||
"""
|
||||
Reasoning parser for Ernie45 thinking model.
|
||||
The Ernie45 thinking model ouput format is
|
||||
The Ernie45 thinking model output format is
|
||||
abc\n</think>\n\n<response>\ndef\n</response>\n
|
||||
or abc\n</think>\ndef
|
||||
"""
|
||||
@@ -73,7 +73,7 @@ class Ernie45ReasoningParser(BaseThinkingReasoningParser):
|
||||
Extract reasoning content from a delta message.
|
||||
Handles streaming output where previous + delta = current.
|
||||
Uses token IDs for faster processing.
|
||||
The Ernie45 thinking model ouput format is
|
||||
The Ernie45 thinking model output format is
|
||||
abc\n</think>\n\n<response>\ndef\n</response>\n
|
||||
or abc\n</think>\ndef
|
||||
- 'abc' goes to reasoning
|
||||
@@ -148,7 +148,7 @@ class Ernie45ReasoningParser(BaseThinkingReasoningParser):
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""
|
||||
Extract reasoning content from the model output.
|
||||
The Ernie45 thinking model ouput format is
|
||||
The Ernie45 thinking model output format is
|
||||
abc\n</think>\n\n\n<response>\ndef\n</response>\n
|
||||
or abc\n</think>\ndef
|
||||
- 'abc' goes to reasoning
|
||||
|
||||
@@ -564,7 +564,7 @@ def replace_vision_chunk_video_placeholder(
|
||||
mm_data: "MultiModalDataDict",
|
||||
video_placeholder: str | None,
|
||||
) -> str | list[int]:
|
||||
# get video placehoder, replace it with runtime video-chunk prompts
|
||||
# get video placeholder, replace it with runtime video-chunk prompts
|
||||
if video_placeholder and isinstance(prompt_raw, str):
|
||||
video_prompts = build_video_prompts_from_mm_data(mm_data)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
Schemas and utilites for preprocessing inputs.
|
||||
Schemas and utilities for preprocessing inputs.
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
Schemas and utilites for tokenization inputs.
|
||||
Schemas and utilities for tokenization inputs.
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
@@ -169,7 +169,7 @@ def _prepare_apply_chat_template_tools_and_messages(
|
||||
tool.pop(tool_key)
|
||||
logger.warning_once(
|
||||
f"'{tool_key}' is not supported by mistral-common for tools. "
|
||||
"It has been poped from the tool definition."
|
||||
"It has been popped from the tool definition."
|
||||
)
|
||||
if tool["type"] == "function":
|
||||
function_keys = list(tool["function"].keys())
|
||||
@@ -178,7 +178,7 @@ def _prepare_apply_chat_template_tools_and_messages(
|
||||
tool["function"].pop(function_key)
|
||||
logger.warning_once(
|
||||
f"'{function_key}' is not supported by mistral-common "
|
||||
"for function tools. It has been poped from the "
|
||||
"for function tools. It has been popped from the "
|
||||
"function definition."
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -402,7 +402,7 @@ class Ovis2_5Processor(ProcessorMixin):
|
||||
images = [images]
|
||||
elif video is not None:
|
||||
is_video = True
|
||||
# type of vidoe in dummy_mm_data is np.ndarray
|
||||
# type of video in dummy_mm_data is np.ndarray
|
||||
if isinstance(video, np.ndarray):
|
||||
images = []
|
||||
for i in range(video.shape[0]):
|
||||
|
||||
@@ -174,7 +174,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
query_start_loc = query_start_loc[: num_decodes + 1]
|
||||
block_table_tensor = block_table_tensor[:num_decodes]
|
||||
|
||||
sheduler_metadata = ops.cpu_attn_get_scheduler_metadata(
|
||||
scheduler_metadata = ops.cpu_attn_get_scheduler_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
@@ -197,7 +197,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
seq_lens=seq_lens,
|
||||
block_table=block_table_tensor,
|
||||
slot_mapping=slot_mapping,
|
||||
scheduler_metadata=sheduler_metadata,
|
||||
scheduler_metadata=scheduler_metadata,
|
||||
causal=causal,
|
||||
use_sdpa_prefill=self.use_sdpa_prefill,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
|
||||
@@ -383,7 +383,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
|
||||
# Return a tensor of shape (#requests, #max blocks)
|
||||
state_indices_tensor = common_attn_metadata.block_table_tensor
|
||||
# Additional cache-related varaiables:
|
||||
# Additional cache-related variables:
|
||||
mamba_block_size = self.kv_cache_spec.block_size
|
||||
(
|
||||
block_idx_last_computed_token,
|
||||
|
||||
@@ -49,14 +49,14 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# For FP8 sparse attention we have two impelementations:
|
||||
# For FP8 sparse attention we have two implementations:
|
||||
# 1. Mixed batch mode: use the FP8 decode kernel for both prefill and decode this is
|
||||
# done by treating all tokens as single batch.
|
||||
# 2. Separate prefill and decode mode: use the BF16 prefill kernel for prefill
|
||||
# (upconverting the FP8 cache to BF16 then calling the prefill kernel) and using
|
||||
# the FP8 decode kernel for decode.
|
||||
# Currently we use #1 when the number of heads per rank is low (i.e. TP) since the BF16
|
||||
# prefill kernel requires padding the numer of heads to 128 while the decode does not
|
||||
# prefill kernel requires padding the number of heads to 128 while the decode does not
|
||||
# so when the per ranke head count is below MIN_HEADS_FOR_BF16_PREFILL we use the mixed
|
||||
# batch mode (#2).
|
||||
MIN_HEADS_FOR_BF16_PREFILL = 32
|
||||
@@ -126,7 +126,7 @@ class FlashMLASparseBackend(AttentionBackend):
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
if cache_dtype_str == "fp8_ds_mla":
|
||||
# custom storage fromat is 656 bytes
|
||||
# custom storage format is 656 bytes
|
||||
# see FlashMLA readme.md for details
|
||||
return (num_blocks, block_size, 656)
|
||||
else:
|
||||
|
||||
@@ -370,7 +370,7 @@ class AiterFlashAttentionMetadata:
|
||||
slot_mapping: torch.Tensor
|
||||
block_table: torch.Tensor
|
||||
|
||||
# prefill and deocde split
|
||||
# prefill and decode split
|
||||
num_decodes: int
|
||||
num_decode_tokens: int
|
||||
num_prefills: int
|
||||
@@ -1099,7 +1099,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
extend_tokens_slice = slice(
|
||||
num_decode_tokens, num_decode_tokens + num_extend_tokens
|
||||
)
|
||||
extend_querys = query[extend_tokens_slice]
|
||||
extend_queries = query[extend_tokens_slice]
|
||||
extend_keys = key[extend_tokens_slice]
|
||||
extend_values = value[extend_tokens_slice]
|
||||
extend_outputs = output[extend_tokens_slice]
|
||||
@@ -1110,7 +1110,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
v_scale = attn_metadata.v_scale
|
||||
self.extend_forward(
|
||||
attn_metadata=attn_metadata,
|
||||
query=extend_querys,
|
||||
query=extend_queries,
|
||||
key=extend_keys,
|
||||
value=extend_values,
|
||||
key_cache=key_cache,
|
||||
|
||||
@@ -863,7 +863,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
):
|
||||
# Mamba can't rely on blocks generated by other requests in the current step
|
||||
# To put it in the next step, we return num_gpu_blocks + 1 so
|
||||
# that kv_cache_manager will think there is no enough blocks to allocte now
|
||||
# that kv_cache_manager will think there is no enough blocks to allocate now
|
||||
# and don't schedule it in the current step.
|
||||
return self.block_pool.num_gpu_blocks + 1
|
||||
if self.mamba_cache_mode != "align":
|
||||
|
||||
@@ -1724,11 +1724,11 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
"""
|
||||
Send notifications to EngineCoreClient, which can then forward
|
||||
the notifications to other engine core processes. It is used for:
|
||||
1) In scale up: new core engines to notify exisiting core engines
|
||||
1) In scale up: new core engines to notify existing core engines
|
||||
that they are ready;
|
||||
2) In scale down: removing core engines to notify EngineCoreClient
|
||||
so EngineCoreClient can release their ray placement groups;
|
||||
3) Both scale up/down: to notify EngineCoreClient that exisiting
|
||||
3) Both scale up/down: to notify EngineCoreClient that existing
|
||||
core engines have already switched to the new parallel setup.
|
||||
"""
|
||||
if vllm_config is None:
|
||||
|
||||
@@ -194,7 +194,7 @@ class InputProcessor:
|
||||
@staticmethod
|
||||
def assign_request_id(request: EngineCoreRequest):
|
||||
"""Replace the externally supplied request ID with an internal request ID
|
||||
that adds 8 random characters in order to ensure uniquness.
|
||||
that adds 8 random characters in order to ensure uniqueness.
|
||||
"""
|
||||
if request.external_req_id is not None:
|
||||
raise ValueError(
|
||||
|
||||
@@ -197,7 +197,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
transfer = self._transfers.popleft()
|
||||
transfer_time = (
|
||||
transfer.start_event.elapsed_time(transfer.end_event) * 1e-3
|
||||
) # elapsed_time is in miliseconds
|
||||
) # elapsed_time is in milliseconds
|
||||
result = TransferResult(
|
||||
job_id=transfer.job_id,
|
||||
success=True,
|
||||
|
||||
@@ -905,7 +905,7 @@ class GPUModelRunner(
|
||||
Args:
|
||||
scheduler_output: The scheduler output.
|
||||
"""
|
||||
# Attention free models have zero kv_cache_goups, however models
|
||||
# Attention free models have zero kv_cache_groups, however models
|
||||
# like Mamba are also attention free but use the kv_cache for
|
||||
# keeping its internal state. This is why we check the number
|
||||
# of kv_cache groups instead of solely checking
|
||||
@@ -1065,7 +1065,7 @@ class GPUModelRunner(
|
||||
# of the request. for example:
|
||||
# fist step: num_computed_tokens = 0, spec_tokens = [],
|
||||
# prev_num_draft_len = 0.
|
||||
# second step: num_computed_tokens = 100(prompt lenth),
|
||||
# second step: num_computed_tokens = 100(prompt length),
|
||||
# spec_tokens = [a,b], prev_num_draft_len = 0.
|
||||
# third step: num_computed_tokens = 100 + 2, spec_tokens = [c,d],
|
||||
# prev_num_draft_len = 2.
|
||||
@@ -1412,30 +1412,30 @@ class GPUModelRunner(
|
||||
prev_draft_token_indices.extend(range(start, start + draft_len))
|
||||
indices_match &= prev_index == flattened_index
|
||||
max_flattened_index = max(max_flattened_index, flattened_index)
|
||||
num_commmon_tokens = len(sample_flattened_indices)
|
||||
num_common_tokens = len(sample_flattened_indices)
|
||||
total_without_spec = total_num_scheduled_tokens - total_num_spec_tokens
|
||||
if num_commmon_tokens < total_without_spec:
|
||||
if num_common_tokens < total_without_spec:
|
||||
# If not all requests are decodes from the last iteration,
|
||||
# We need to copy the input_ids_cpu to the GPU first.
|
||||
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||
if self.enable_prompt_embeds:
|
||||
self.inputs_embeds.copy_to_gpu(total_num_scheduled_tokens)
|
||||
self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||
if num_commmon_tokens == 0:
|
||||
if num_common_tokens == 0:
|
||||
# No requests in common with the previous iteration
|
||||
# So input_ids.cpu will have all the input ids.
|
||||
return
|
||||
if indices_match and max_flattened_index == (num_commmon_tokens - 1):
|
||||
if indices_match and max_flattened_index == (num_common_tokens - 1):
|
||||
# Common-case optimization: the batch is unchanged
|
||||
# and no reordering happened.
|
||||
# The indices are both the same permutation of 0..N-1 so
|
||||
# we can copy directly using a single slice.
|
||||
self.input_ids.gpu[:num_commmon_tokens].copy_(
|
||||
self.input_batch.prev_sampled_token_ids[:num_commmon_tokens, 0],
|
||||
self.input_ids.gpu[:num_common_tokens].copy_(
|
||||
self.input_batch.prev_sampled_token_ids[:num_common_tokens, 0],
|
||||
non_blocking=True,
|
||||
)
|
||||
if self.enable_prompt_embeds:
|
||||
self.is_token_ids.gpu[:num_commmon_tokens] = True
|
||||
self.is_token_ids.gpu[:num_common_tokens] = True
|
||||
return
|
||||
# Upload the index tensors asynchronously so the scatter can be non-blocking.
|
||||
sampled_tokens_index_tensor = torch.tensor(
|
||||
@@ -4383,7 +4383,7 @@ class GPUModelRunner(
|
||||
self.model.compile(fullgraph=True, backend=backend)
|
||||
return
|
||||
# for other compilation modes, cudagraph behavior is controlled by
|
||||
# CudagraphWraper and CudagraphDispatcher of vllm.
|
||||
# CudagraphWrapper and CudagraphDispatcher of vllm.
|
||||
|
||||
# wrap the model with full cudagraph wrapper if needed.
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode
|
||||
@@ -4444,7 +4444,7 @@ class GPUModelRunner(
|
||||
:param weights_path: path to load weights from if weights_iterator is not
|
||||
provided. Use path of original model if neither is provided.
|
||||
:param is_checkpoint_format: set to False if weights have already been processed
|
||||
into kernel format (repacking, renaming, ect.)
|
||||
into kernel format (repacking, renaming, etc.)
|
||||
"""
|
||||
# TODO(@kylesayrs): generalize to all runners and loaders
|
||||
# argument validation
|
||||
|
||||
Reference in New Issue
Block a user