[CPU] Add MLA backend so DeepSeek-V2/V3 can run on CPU (#49453)
Signed-off-by: baoloongmao <baoloongmao@tencent.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
ad5280255b
commit
13726c80fe
@@ -943,10 +943,14 @@ inline FP16Vec16::FP16Vec16(const FP32Vec16& v) {
|
||||
};
|
||||
|
||||
inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
|
||||
fmadd(acc.reg.val[0], a.reg.val[0], b.reg.val[0]);
|
||||
fmadd(acc.reg.val[1], a.reg.val[1], b.reg.val[1]);
|
||||
fmadd(acc.reg.val[2], a.reg.val[2], b.reg.val[2]);
|
||||
fmadd(acc.reg.val[3], a.reg.val[3], b.reg.val[3]);
|
||||
// NOTE: at::vec::fmadd(a, b, c) returns a * b + c and does not mutate its
|
||||
// arguments; we must capture the result back into acc, otherwise the FMA
|
||||
// is a no-op. On x86 fma is implemented as `acc = acc + a * b` which does
|
||||
// the assignment; the ARM version previously dropped the return value.
|
||||
acc.reg.val[0] = fmadd(a.reg.val[0], b.reg.val[0], acc.reg.val[0]);
|
||||
acc.reg.val[1] = fmadd(a.reg.val[1], b.reg.val[1], acc.reg.val[1]);
|
||||
acc.reg.val[2] = fmadd(a.reg.val[2], b.reg.val[2], acc.reg.val[2]);
|
||||
acc.reg.val[3] = fmadd(a.reg.val[3], b.reg.val[3], acc.reg.val[3]);
|
||||
};
|
||||
|
||||
inline BF16Vec8::BF16Vec8(const FP32Vec8& v) {
|
||||
@@ -967,10 +971,11 @@ inline void fma(FP32Vec16& acc, BF16Vec32& a, BF16Vec32& b) {
|
||||
std::tie(b0_low, b0_high) = convert_bfloat16_float(b.reg.val[0]);
|
||||
std::tie(b1_low, b1_high) = convert_bfloat16_float(b.reg.val[1]);
|
||||
|
||||
fmadd(acc.reg.val[0], a0_low, b0_low);
|
||||
fmadd(acc.reg.val[1], a0_high, b0_high);
|
||||
fmadd(acc.reg.val[2], a1_low, b1_low);
|
||||
fmadd(acc.reg.val[3], a1_high, b1_high);
|
||||
// Same fmadd semantic as the FP32 overload above: capture the result.
|
||||
acc.reg.val[0] = fmadd(a0_low, b0_low, acc.reg.val[0]);
|
||||
acc.reg.val[1] = fmadd(a0_high, b0_high, acc.reg.val[1]);
|
||||
acc.reg.val[2] = fmadd(a1_low, b1_low, acc.reg.val[2]);
|
||||
acc.reg.val[3] = fmadd(a1_high, b1_high, acc.reg.val[3]);
|
||||
};
|
||||
|
||||
template <>
|
||||
|
||||
@@ -57,6 +57,12 @@ void mla_decode_block_head(
|
||||
std::fill(max_val, max_val + HEAD_UNROLL, -FLT_MAX);
|
||||
|
||||
f32_vec_type acc_vec[BLOCK_SIZE][HEAD_UNROLL];
|
||||
// Explicitly zero-initialize the accumulator. On x86 the default
|
||||
// ctor of FP32Vec16 already sets zero, but on ARM (and any backend
|
||||
// whose Vectorized<float> default ctor is trivial) the array holds
|
||||
// undefined values and the fma below accumulates garbage.
|
||||
for (int b = 0; b < BLOCK_SIZE; ++b)
|
||||
for (int u = 0; u < HEAD_UNROLL; ++u) acc_vec[b][u] = f32_vec_type{0.0f};
|
||||
for (int i = 0; i < HEAD_DIM; i += QK_NUM_ELEM) {
|
||||
// load to registers
|
||||
qk_vec_type q_vec[HEAD_UNROLL];
|
||||
@@ -96,6 +102,9 @@ void mla_decode_block_head(
|
||||
}
|
||||
|
||||
f32_vec_type this_out[V_HEAD_DIM / V_NUM_ELEM][HEAD_UNROLL];
|
||||
// Same zero-initialization concern as acc_vec above.
|
||||
for (int i = 0; i < V_HEAD_DIM / V_NUM_ELEM; ++i)
|
||||
for (int u = 0; u < HEAD_UNROLL; ++u) this_out[i][u] = f32_vec_type{0.0f};
|
||||
|
||||
for (int block_offset = 0; block_offset < num_tokens; ++block_offset) {
|
||||
// load to registers
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Smoke tests for the CPU MLA backend.
|
||||
|
||||
Two levels of coverage are provided:
|
||||
|
||||
* :func:`test_kv_cache_cpu_write` checks that the kv-cache write logic
|
||||
inlined in the CPU MLA backend writes the latent KV cache in the same
|
||||
layout the CPU decode kernel expects. This runs without downloading
|
||||
any model weights and is safe for CI.
|
||||
* :func:`test_cpu_mla_backend_smoke` exercises the full end-to-end path
|
||||
(LLM engine, model construction, prefill + decode) against a shrunk
|
||||
DeepSeek-V2-Lite with `hf_overrides`. It uses `dummy` weights so the
|
||||
outputs are not meaningful, only shape / plumbing correctness is
|
||||
asserted. Marked `cpu_model` so it is skipped on non-CPU CI.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
def test_kv_cache_cpu_write() -> None:
|
||||
kv_lora_rank = 512
|
||||
pe_dim = 64
|
||||
block_size = 16
|
||||
num_blocks = 4
|
||||
num_tokens = 8
|
||||
dtype = torch.float32
|
||||
|
||||
kv_c = torch.randn(num_tokens, kv_lora_rank, dtype=dtype)
|
||||
k_pe = torch.randn(num_tokens, pe_dim, dtype=dtype)
|
||||
kv_cache = torch.zeros(num_blocks, block_size, kv_lora_rank + pe_dim, dtype=dtype)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.long)
|
||||
# Mark the last slot as padding to make sure we honour negative slots.
|
||||
slot_mapping[-1] = -1
|
||||
|
||||
flat = kv_cache.view(-1, kv_lora_rank + pe_dim)
|
||||
slots = slot_mapping.to(torch.long)
|
||||
valid_mask = slots >= 0
|
||||
if not valid_mask.all().item():
|
||||
slots = slots[valid_mask]
|
||||
kv_c = kv_c[valid_mask]
|
||||
k_pe = k_pe[valid_mask]
|
||||
target_dtype = flat.dtype
|
||||
flat[slots, :kv_lora_rank] = kv_c.to(target_dtype)
|
||||
flat[slots, kv_lora_rank:] = k_pe.to(target_dtype)
|
||||
|
||||
flat = kv_cache.view(-1, kv_lora_rank + pe_dim)
|
||||
for i in range(num_tokens - 1):
|
||||
assert torch.equal(flat[i, :kv_lora_rank], kv_c[i])
|
||||
assert torch.equal(flat[i, kv_lora_rank:], k_pe[i])
|
||||
# The padding slot must remain untouched.
|
||||
assert torch.all(flat[num_tokens - 1] == 0)
|
||||
|
||||
|
||||
@pytest.mark.cpu_model
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
def test_cpu_mla_backend_smoke() -> None:
|
||||
# Shrink the architecture aggressively so the test is CI-friendly.
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
hf_overrides = {
|
||||
"num_hidden_layers": 2,
|
||||
"n_routed_experts": 4,
|
||||
"first_k_dense_replace": 0,
|
||||
"num_experts_per_tok": 2,
|
||||
}
|
||||
llm = LLM(
|
||||
model="deepseek-ai/DeepSeek-V2-Lite",
|
||||
trust_remote_code=True,
|
||||
load_format="dummy",
|
||||
enforce_eager=True,
|
||||
max_model_len=128,
|
||||
max_num_seqs=2,
|
||||
block_size=16,
|
||||
hf_overrides=hf_overrides,
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=4, temperature=0.0)
|
||||
outputs = llm.generate(["Hello", "MLA on CPU"], sampling_params)
|
||||
assert len(outputs) == 2
|
||||
for output in outputs:
|
||||
# `dummy` weights do not produce meaningful text, but the number
|
||||
# of generated tokens must match what we asked for.
|
||||
assert len(output.outputs[0].token_ids) == 4
|
||||
+36
-6
@@ -78,12 +78,19 @@ class CpuPlatform(Platform):
|
||||
attn_selector_config: "AttentionSelectorConfig",
|
||||
num_heads: int | None = None,
|
||||
) -> str:
|
||||
if selected_backend and selected_backend != AttentionBackendEnum.CPU_ATTN:
|
||||
logger.info("Cannot use %s backend on CPU.", selected_backend)
|
||||
if attn_selector_config.use_mla:
|
||||
raise NotImplementedError("MLA is not supported on CPU.")
|
||||
if attn_selector_config.use_sparse:
|
||||
raise NotImplementedError("Sparse Attention is not supported on CPU.")
|
||||
if attn_selector_config.use_mla:
|
||||
# Reference MLA implementation on CPU. Performance is not the
|
||||
# goal here; the backend simply wires the CPU decode kernel
|
||||
# (`mla_decode_kvcache`) and an SDPA-based prefill together with
|
||||
# the shared MLA scaffolding so that DeepSeek-style models can
|
||||
# execute on CPU.
|
||||
if selected_backend and selected_backend != AttentionBackendEnum.CPU_MLA:
|
||||
logger.info("Cannot use %s backend on CPU.", selected_backend)
|
||||
return AttentionBackendEnum.CPU_MLA.get_path()
|
||||
if selected_backend and selected_backend != AttentionBackendEnum.CPU_ATTN:
|
||||
logger.info("Cannot use %s backend on CPU.", selected_backend)
|
||||
return AttentionBackendEnum.CPU_ATTN.get_path()
|
||||
|
||||
@classmethod
|
||||
@@ -116,10 +123,25 @@ class CpuPlatform(Platform):
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
if not cache_config.user_specified_block_size:
|
||||
# The CPU MLA decode kernel only compiles with block_size=16 today
|
||||
# (see csrc/cpu/mla_decode.cpp). If the model uses MLA we override
|
||||
# the default block size regardless of user preference to avoid a
|
||||
# runtime kernel dispatch failure.
|
||||
cpu_mla_enabled = model_config is not None and getattr(
|
||||
model_config, "use_mla", False
|
||||
)
|
||||
if cpu_mla_enabled:
|
||||
if cache_config.user_specified_block_size and cache_config.block_size != 16:
|
||||
logger.warning(
|
||||
"CPU MLA backend requires block_size=16, overriding "
|
||||
"user-specified block_size=%s.",
|
||||
cache_config.block_size,
|
||||
)
|
||||
cache_config.block_size = 16
|
||||
elif not cache_config.user_specified_block_size:
|
||||
cache_config.block_size = 128
|
||||
|
||||
if cache_config.block_size % 32 != 0:
|
||||
if not cpu_mla_enabled and cache_config.block_size % 32 != 0:
|
||||
logger.warning(
|
||||
"CPU backend prefers block_size is multiples of 32, "
|
||||
"otherwise the performance is not optimized."
|
||||
@@ -321,6 +343,7 @@ class CpuPlatform(Platform):
|
||||
"prefill and prefix caching to be disabled."
|
||||
)
|
||||
vllm_config.scheduler_config.enable_chunked_prefill = False
|
||||
vllm_config.cache_config.enable_prefix_caching = False
|
||||
vllm_config.scheduler_config.max_num_batched_tokens = max(
|
||||
vllm_config.model_config.max_model_len,
|
||||
vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS,
|
||||
@@ -473,6 +496,13 @@ class CpuPlatform(Platform):
|
||||
from vllm._custom_ops import cpu_attn_reshape_and_cache
|
||||
from vllm.v1.attention.backends.cpu_attn import _get_attn_isa
|
||||
|
||||
# MLA uses a single latent cache of shape [N, block_size, head_size],
|
||||
# so the classic key/value split does not apply. The MLA backend
|
||||
# writes the cache itself via `concat_and_cache_mla` inside
|
||||
# `do_kv_cache_update`, so there is nothing to pack here.
|
||||
if kv_cache.dim() == 3:
|
||||
return
|
||||
|
||||
num_blocks, num_kv_heads, block_size, fused_head_size = kv_cache.shape
|
||||
head_size = fused_head_size // 2
|
||||
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU MLA backend.
|
||||
|
||||
This is a *reference-quality* MLA backend for the CPU platform. It is
|
||||
intended to make DeepSeek-V2/V3 style models runnable on CPU (for
|
||||
functional verification, tiny-model smoke tests and CI), not to be
|
||||
performant. The prefill path is a plain PyTorch SDPA and the decode
|
||||
path forwards to the existing CPU decode kernel
|
||||
`torch.ops._C.mla_decode_kvcache` (see csrc/cpu/mla_decode.cpp).
|
||||
|
||||
Key design points:
|
||||
|
||||
* We inherit the shared MLA scaffolding (`MLACommonBackend`,
|
||||
`MLACommonImpl`, `MLACommonMetadata`) so that the same
|
||||
``forward_impl`` in ``MLAAttention`` orchestrates weight-absorbed
|
||||
decode (MQA) and non-absorbed prefill (MHA) for us.
|
||||
* The parent ``MLACommonImpl.__init__`` tries to pick a GPU prefill
|
||||
kernel (flash_attn / flashinfer / cudnn / trtllm) and raises when
|
||||
none of them are available. On CPU none of them apply, so we
|
||||
bypass that logic and set the attributes ourselves.
|
||||
* Chunked prefill and prefix caching are disabled by the CPU platform
|
||||
when ``use_mla`` is set.
|
||||
* The CPU decode kernel only supports ``head_dim=576``,
|
||||
``v_head_dim=512`` and ``block_size=16`` today; the platform layer
|
||||
forces ``block_size=16`` for MLA models.
|
||||
"""
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBackend,
|
||||
MLACommonImpl,
|
||||
MLACommonMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionLayer,
|
||||
AttentionType,
|
||||
MultipleOf,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CPUMLABackend(MLACommonBackend):
|
||||
"""Attention backend descriptor for CPU MLA."""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
torch.float32,
|
||||
]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto"]
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
# DeepSeek-V2/V3 latent cache size = kv_lora_rank(512)+rope(64)=576.
|
||||
# The current CPU decode kernel only compiles for this size.
|
||||
return [576]
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [16]
|
||||
|
||||
@classmethod
|
||||
def supports_block_size(cls, block_size: int | None) -> bool:
|
||||
return block_size is None or block_size == 16
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CPU_MLA"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["CPUMLAImpl"]:
|
||||
return CPUMLAImpl
|
||||
|
||||
|
||||
class CPUMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
"""CPU implementation of MLA attention.
|
||||
|
||||
See module docstring for the overall design. This class only
|
||||
overrides the parts of the parent that would otherwise pull in
|
||||
CUDA-only dependencies.
|
||||
"""
|
||||
|
||||
# Decode always returns the softmax lse if the parent needs it, but
|
||||
# CPU decode currently only produces the output tensor. Report False
|
||||
# so DCP / cross-batch reductions are not requested from us.
|
||||
can_return_lse_for_decode: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: float | None,
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA-specific arguments (forwarded from `MLAAttention`).
|
||||
**mla_args,
|
||||
) -> None:
|
||||
# NOTE: We deliberately skip ``MLACommonImpl.__init__`` because it
|
||||
# tries to pick a GPU prefill kernel and raises on CPU. Instead we
|
||||
# replicate the attribute setup here and then wire our own hooks.
|
||||
if kv_sharing_target_layer_name is not None:
|
||||
raise NotImplementedError("KV sharing is not supported for MLA")
|
||||
if attn_type != AttentionType.DECODER:
|
||||
raise NotImplementedError("CPU MLA only supports decoder self-attention.")
|
||||
unsupported = [alibi_slopes, sliding_window, logits_soft_cap]
|
||||
if any(unsupported):
|
||||
raise NotImplementedError(
|
||||
"CPU MLA does not support alibi_slopes / sliding_window / "
|
||||
"logits_soft_cap."
|
||||
)
|
||||
|
||||
self.num_heads = num_heads
|
||||
self.head_size = head_size
|
||||
self.scale = float(scale)
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
|
||||
self.q_lora_rank = mla_args["q_lora_rank"]
|
||||
self.kv_lora_rank = mla_args["kv_lora_rank"]
|
||||
self.qk_nope_head_dim = mla_args["qk_nope_head_dim"]
|
||||
self.qk_rope_head_dim = mla_args["qk_rope_head_dim"]
|
||||
self.qk_head_dim = mla_args["qk_head_dim"]
|
||||
self.v_head_dim = mla_args["v_head_dim"]
|
||||
self.kv_b_proj = mla_args["kv_b_proj"]
|
||||
self.indexer = mla_args.get("indexer")
|
||||
self.q_pad_num_heads = mla_args.get("q_pad_num_heads")
|
||||
self.supports_quant_query_input = False
|
||||
|
||||
# Never take the flashinfer specialisation of ``_concat_k_nope_k_pe``.
|
||||
self._use_flashinfer_concat_mla_k = False
|
||||
|
||||
# We do not need V padding since our SDPA prefill takes k / v with
|
||||
# matching head dims.
|
||||
self._pad_v = False
|
||||
|
||||
# Distributed / context-parallel bookkeeping expected by the parent.
|
||||
self.dcp_world_size = 1
|
||||
self.cp_kv_cache_interleave_size = 1
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Decode (MQA)
|
||||
# ------------------------------------------------------------------
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: MLACommonMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert attn_metadata.decode is not None
|
||||
# The parent hands us either a tuple (q_nope, q_pe) or a single
|
||||
# tensor already concatenated. CPU decode kernel wants a single
|
||||
# (bs, num_heads, head_dim=kv_lora_rank+rope) contiguous tensor.
|
||||
if isinstance(q, tuple):
|
||||
q = torch.cat(q, dim=-1)
|
||||
q = q.contiguous()
|
||||
|
||||
bs, num_heads, _ = q.shape
|
||||
out = torch.zeros(
|
||||
bs, num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device
|
||||
)
|
||||
|
||||
# The CPU kernel is templated on int32 block_tables / seq_lens.
|
||||
block_table = attn_metadata.decode.block_table.to(torch.int32)
|
||||
seq_lens = attn_metadata.decode.seq_lens.to(torch.int32)
|
||||
|
||||
ops.mla_decode_kvcache_cpu(
|
||||
out,
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
self.scale,
|
||||
block_table,
|
||||
seq_lens,
|
||||
)
|
||||
# The parent only consumes the LSE when DCP is active, which we
|
||||
# disable, so returning None here is safe.
|
||||
return out, None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# KV cache write
|
||||
# ------------------------------------------------------------------
|
||||
def do_kv_cache_update(
|
||||
self,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
) -> None:
|
||||
if kv_cache.numel() == 0:
|
||||
return
|
||||
k_pe_sq = k_pe.squeeze(1)
|
||||
kv_lora_rank = kv_c_normed.shape[-1]
|
||||
pe_dim = k_pe_sq.shape[-1]
|
||||
flat = kv_cache.view(-1, kv_lora_rank + pe_dim)
|
||||
slots = slot_mapping.flatten().to(torch.long)
|
||||
# Only write valid slots. Padding slots are set to -1 by the
|
||||
# scheduler and must be skipped.
|
||||
valid_mask = slots >= 0
|
||||
if not valid_mask.all().item():
|
||||
slots = slots[valid_mask]
|
||||
kv_c_normed = kv_c_normed[valid_mask]
|
||||
k_pe_sq = k_pe_sq[valid_mask]
|
||||
target_dtype = flat.dtype
|
||||
flat[slots, :kv_lora_rank] = kv_c_normed.to(target_dtype)
|
||||
flat[slots, kv_lora_rank:] = k_pe_sq.view(-1, pe_dim).to(target_dtype)
|
||||
@@ -122,6 +122,7 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
"RocmAiterUnifiedAttentionBackend"
|
||||
)
|
||||
CPU_ATTN = "vllm.v1.attention.backends.cpu_attn.CPUAttentionBackend"
|
||||
CPU_MLA = "vllm.v1.attention.backends.mla.cpu_mla.CPUMLABackend"
|
||||
TURBOQUANT = "vllm.v1.attention.backends.turboquant_attn.TurboQuantAttentionBackend"
|
||||
# Placeholder for third-party/custom backends - must be registered before use
|
||||
# set to None to avoid alias with other backend, whose value is an empty string
|
||||
|
||||
Reference in New Issue
Block a user