[Model] Add LongCat-Flash-Lite (n-gram embedding) (#47857)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -390,6 +390,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
"csrc/libtorch_stable/cuda_view.cu"
|
||||
"csrc/libtorch_stable/cuda_utils_kernels.cu"
|
||||
"csrc/libtorch_stable/activation_kernels.cu"
|
||||
"csrc/libtorch_stable/ngram_embedding_kernels.cu"
|
||||
"csrc/libtorch_stable/quantization/activation_kernels.cu"
|
||||
"csrc/libtorch_stable/quantization/w8a8/int8/scaled_quant.cu"
|
||||
"csrc/libtorch_stable/quantization/w8a8/fp8/common.cu"
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
// N-gram embedding index kernel for LongCat-Flash (n-gram embedding variant).
|
||||
//
|
||||
// Adapted from SGLang:
|
||||
// https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/csrc/ngram_embedding.cuh
|
||||
//
|
||||
// For each position, computes the hashed n-gram embedding ids that index the
|
||||
// concatenated embedder table. Integer tensors are int32 except ``row_indices``
|
||||
// (int64); the token table is ``[max_running_reqs, max_context_len]`` int32,
|
||||
// where a negative entry marks an ignored token (e.g. an EOS boundary).
|
||||
|
||||
#include "torch_utils.h"
|
||||
|
||||
#include "ops.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace vllm::ngram_embedding {
|
||||
|
||||
constexpr int kBlockThreads = 256;
|
||||
|
||||
__global__ void ComputeNGramIdsKernel(
|
||||
int batch_size, int ne_n, int ne_k,
|
||||
int* ne_weights, // [ne_n-1, ne_k, ne_n]
|
||||
int* ne_mods, // [ne_n-1, ne_k]
|
||||
int* exclusive_ne_embedder_size_sums, // [(ne_n-1)*ne_k + 1]
|
||||
int* exclusive_req_len_sums, // [batch_size + 1]
|
||||
int* ne_token_table, // [max_running_reqs, max_context_len]
|
||||
int max_context_len,
|
||||
const int64_t* __restrict__ row_indices, // [batch_size]
|
||||
int* column_starts, // [batch_size]
|
||||
int* n_gram_ids // [token_num, (ne_n-1)*ne_k]
|
||||
) {
|
||||
const int req_id = blockIdx.x % batch_size;
|
||||
const int config_id = (blockIdx.x - req_id) / batch_size;
|
||||
// n and k are offset from their physical meaning: n = real_n - 2, k = real_k
|
||||
// - 1 (they index into ne_weights / ne_mods).
|
||||
const int k = config_id % ne_k;
|
||||
const int n = (config_id - config_id % ne_k) / ne_k;
|
||||
const int ne_weight_base_idx = n * ne_k * ne_n + k * ne_n;
|
||||
const int ne_mod = ne_mods[n * ne_k + k];
|
||||
for (int i = exclusive_req_len_sums[req_id] + threadIdx.x;
|
||||
i < exclusive_req_len_sums[req_id + 1]; i += blockDim.x) {
|
||||
uint64_t n_gram_id = 0;
|
||||
const int64_t current_token_offset = i - exclusive_req_len_sums[req_id];
|
||||
const int64_t req_token_table_index =
|
||||
row_indices[req_id] * static_cast<int64_t>(max_context_len);
|
||||
const int64_t current_token_table_index =
|
||||
req_token_table_index + column_starts[req_id] + current_token_offset;
|
||||
for (int j = 0; j < n + 2; j++) {
|
||||
if (current_token_table_index - j < req_token_table_index) {
|
||||
break; // outside this request's range
|
||||
}
|
||||
if (ne_token_table[current_token_table_index - j] < 0) {
|
||||
break; // ignored token
|
||||
}
|
||||
const uint64_t term =
|
||||
(uint64_t)ne_token_table[current_token_table_index - j] *
|
||||
(uint64_t)ne_weights[ne_weight_base_idx + j];
|
||||
n_gram_id += term % ne_mod;
|
||||
}
|
||||
n_gram_id %= ne_mod;
|
||||
n_gram_id += exclusive_ne_embedder_size_sums[n * ne_k + k];
|
||||
n_gram_ids[i * (ne_n - 1) * ne_k + n * ne_k + k] = (int)(n_gram_id);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace vllm::ngram_embedding
|
||||
|
||||
void ngram_compute_n_gram_ids(
|
||||
int64_t ne_n, int64_t ne_k, torch::stable::Tensor& ne_weights,
|
||||
torch::stable::Tensor& ne_mods,
|
||||
torch::stable::Tensor& exclusive_ne_embedder_size_sums,
|
||||
torch::stable::Tensor& exclusive_req_len_sums,
|
||||
torch::stable::Tensor& ne_token_table, torch::stable::Tensor& row_indices,
|
||||
torch::stable::Tensor& column_starts, torch::stable::Tensor& n_gram_ids) {
|
||||
const int batch_size = static_cast<int>(exclusive_req_len_sums.size(0) - 1);
|
||||
const int max_context_len = static_cast<int>(ne_token_table.size(1));
|
||||
const int num_configs = (static_cast<int>(ne_n) - 1) * static_cast<int>(ne_k);
|
||||
const int grid_size = num_configs * batch_size;
|
||||
if (grid_size <= 0) return;
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
ne_weights.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
vllm::ngram_embedding::ComputeNGramIdsKernel<<<
|
||||
grid_size, vllm::ngram_embedding::kBlockThreads, 0, stream>>>(
|
||||
batch_size, static_cast<int>(ne_n), static_cast<int>(ne_k),
|
||||
ne_weights.mutable_data_ptr<int32_t>(),
|
||||
ne_mods.mutable_data_ptr<int32_t>(),
|
||||
exclusive_ne_embedder_size_sums.mutable_data_ptr<int32_t>(),
|
||||
exclusive_req_len_sums.mutable_data_ptr<int32_t>(),
|
||||
ne_token_table.mutable_data_ptr<int32_t>(), max_context_len,
|
||||
row_indices.const_data_ptr<int64_t>(),
|
||||
column_starts.mutable_data_ptr<int32_t>(),
|
||||
n_gram_ids.mutable_data_ptr<int32_t>());
|
||||
}
|
||||
@@ -554,3 +554,12 @@ void cp_gather_indexer_k_quant_cache(
|
||||
// quant_block_size * 4]
|
||||
const torch::stable::Tensor& block_table, // [batch_size, num_blocks]
|
||||
const torch::stable::Tensor& cu_seq_lens); // [batch_size + 1]
|
||||
|
||||
// LongCat n-gram embedding index kernel (see ngram_embedding_kernels.cu).
|
||||
void ngram_compute_n_gram_ids(
|
||||
int64_t ne_n, int64_t ne_k, torch::stable::Tensor& ne_weights,
|
||||
torch::stable::Tensor& ne_mods,
|
||||
torch::stable::Tensor& exclusive_ne_embedder_size_sums,
|
||||
torch::stable::Tensor& exclusive_req_len_sums,
|
||||
torch::stable::Tensor& ne_token_table, torch::stable::Tensor& row_indices,
|
||||
torch::stable::Tensor& column_starts, torch::stable::Tensor& n_gram_ids);
|
||||
|
||||
@@ -598,9 +598,22 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor? initial_state_idx,"
|
||||
"Tensor? cu_chunk_seqlen,"
|
||||
"Tensor? last_chunk_indices) -> ()");
|
||||
|
||||
// LongCat n-gram embedding index kernel. All tensor args are marked mutable
|
||||
// to match the (non-const) stable-Tensor& C++ signature; only ne_token_table
|
||||
// and n_gram_ids are actually written in place.
|
||||
ops.def(
|
||||
"ngram_compute_n_gram_ids(int ne_n, int ne_k, Tensor(a!) ne_weights, "
|
||||
"Tensor(b!) ne_mods, Tensor(c!) exclusive_ne_embedder_size_sums, "
|
||||
"Tensor(d!) exclusive_req_len_sums, Tensor(e!) ne_token_table, "
|
||||
"Tensor(f!) row_indices, Tensor(g!) column_starts, "
|
||||
"Tensor(h!) n_gram_ids) -> ()");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
// LongCat n-gram embedding index kernel.
|
||||
ops.impl("ngram_compute_n_gram_ids", TORCH_BOX(&ngram_compute_n_gram_ids));
|
||||
|
||||
// Per-token group quantization
|
||||
ops.impl("per_token_group_fp8_quant", TORCH_BOX(&per_token_group_quant_fp8));
|
||||
ops.impl("per_token_group_fp8_quant_packed",
|
||||
|
||||
@@ -382,6 +382,13 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"LongcatFlashForCausalLM": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
|
||||
),
|
||||
"LongcatFlashNgramForCausalLM": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Lite",
|
||||
trust_remote_code=True,
|
||||
# Shrink the ~62GB n-gram tables (ngram_vocab_size_ratio * vocab_size)
|
||||
# so the dummy-weight init test fits in CI memory.
|
||||
hf_overrides={"ngram_vocab_size_ratio": 1},
|
||||
),
|
||||
"MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
|
||||
"Mamba2ForCausalLM": _HfExamplesInfo(
|
||||
"mistralai/Mamba-Codestral-7B-v0.1",
|
||||
|
||||
@@ -529,8 +529,13 @@ def dummy_hf_overrides(
|
||||
}
|
||||
)
|
||||
|
||||
# Update num_hidden_layers for non-Longcat architectures
|
||||
if model_arch != "LongcatFlashForCausalLM" and model_arch != "LongCatFlashMTPModel":
|
||||
# Update num_hidden_layers for non-Longcat architectures (Longcat derives it
|
||||
# from num_layers for its dual-attention layers).
|
||||
if model_arch not in (
|
||||
"LongcatFlashForCausalLM",
|
||||
"LongCatFlashMTPModel",
|
||||
"LongcatFlashNgramForCausalLM",
|
||||
):
|
||||
update_dict["num_hidden_layers"] = num_hidden_layers
|
||||
|
||||
text_config.update(update_dict)
|
||||
|
||||
@@ -235,6 +235,37 @@ def rms_norm(
|
||||
torch.ops._C.rms_norm(out, input, weight, epsilon)
|
||||
|
||||
|
||||
# LongCat n-gram embedding index kernel (see csrc/.../ngram_embedding_kernels.cu).
|
||||
def ngram_compute_n_gram_ids(
|
||||
ne_n: int,
|
||||
ne_k: int,
|
||||
ne_weights: torch.Tensor,
|
||||
ne_mods: torch.Tensor,
|
||||
exclusive_ne_embedder_size_sums: torch.Tensor,
|
||||
exclusive_req_len_sums: torch.Tensor,
|
||||
ne_token_table: torch.Tensor,
|
||||
row_indices: torch.Tensor,
|
||||
column_starts: torch.Tensor,
|
||||
n_gram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
"""Compute concatenated (offset) n-gram ids for a ragged prefill batch.
|
||||
|
||||
Writes ``n_gram_ids`` of shape ``[token_num, (ne_n-1)*ne_k]``.
|
||||
"""
|
||||
torch.ops._C.ngram_compute_n_gram_ids(
|
||||
ne_n,
|
||||
ne_k,
|
||||
ne_weights,
|
||||
ne_mods,
|
||||
exclusive_ne_embedder_size_sums,
|
||||
exclusive_req_len_sums,
|
||||
ne_token_table,
|
||||
row_indices,
|
||||
column_starts,
|
||||
n_gram_ids,
|
||||
)
|
||||
|
||||
|
||||
def fused_add_rms_norm(
|
||||
input: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
|
||||
@@ -518,7 +518,7 @@ class SpeculativeConfig:
|
||||
"architectures": ["Qwen3_5MoeMTP" if is_moe else "Qwen3_5MTP"],
|
||||
}
|
||||
)
|
||||
if hf_config.model_type == "longcat_flash":
|
||||
if hf_config.model_type in ("longcat_flash", "longcat_flash_ngram"):
|
||||
hf_config.model_type = "longcat_flash_mtp"
|
||||
n_predict = getattr(hf_config, "num_nextn_predict_layers", 1)
|
||||
hf_config.update(
|
||||
|
||||
@@ -70,6 +70,7 @@ DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
|
||||
"DeepseekV2ForCausalLM",
|
||||
"Qwen2MoeForCausalLM",
|
||||
"GraniteMoeForCausalLM",
|
||||
"LongcatFlashNgramForCausalLM",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -499,7 +499,7 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig):
|
||||
"last": "LAST",
|
||||
}
|
||||
|
||||
pooling_type = pooling_type_map.get(hf_config.pooling, None)
|
||||
pooling_type = pooling_type_map.get(hf_config.pooling)
|
||||
if pooling_type is None:
|
||||
raise ValueError(f"pool_type {hf_config.pooling!r} not supported")
|
||||
|
||||
@@ -809,6 +809,23 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig):
|
||||
model_config.hf_config.embedding_size = model_config.hf_config.num_labels
|
||||
|
||||
|
||||
class LongcatFlashNgramForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
# LongCat-Flash-Lite's zero-expert MoE trips a data-dependent assert
|
||||
# under torch.compile, and its n-gram inputs_embeds are only wired for
|
||||
# FULL cudagraph capture (PIECEWISE prefill drops them). Default to
|
||||
# no-compile + FULL cudagraph (prefill runs eager) unless the user
|
||||
# configured compilation explicitly.
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if compilation_config.mode is None:
|
||||
compilation_config.mode = CompilationMode.NONE
|
||||
if compilation_config.cudagraph_mode is None:
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.FULL
|
||||
|
||||
|
||||
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
|
||||
"ColQwen3_5": ColQwen3_5Config,
|
||||
@@ -822,6 +839,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"Gemma4ForConditionalGeneration": Gemma4Config,
|
||||
"Gemma4UnifiedForConditionalGeneration": Gemma4Config,
|
||||
"GptOssForCausalLM": GptOssForCausalLMConfig,
|
||||
"LongcatFlashNgramForCausalLM": LongcatFlashNgramForCausalLMConfig,
|
||||
"GteModel": SnowflakeGteNewModelConfig,
|
||||
"GteNewForSequenceClassification": GteNewModelConfig,
|
||||
"GteNewModel": GteNewModelConfig,
|
||||
|
||||
@@ -318,7 +318,7 @@ class LongcatMoe(nn.Module):
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
# Align to FusedMoE padded hidden size to avoid dim mismatch
|
||||
padded_hidden = self.experts.hidden_size
|
||||
padded_hidden = self.experts.moe_config.hidden_dim
|
||||
if hidden_dim < padded_hidden:
|
||||
hidden_states_padded = torch.nn.functional.pad(
|
||||
hidden_states,
|
||||
@@ -687,14 +687,23 @@ class FlashModel(nn.Module):
|
||||
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
|
||||
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
|
||||
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
|
||||
if self.config.mla_scale_q_lora:
|
||||
# Guard against compounding on incremental load_weights calls:
|
||||
# the in-place ``*=`` would otherwise re-apply the MLA LoRA
|
||||
# scaling to the layernorm weights on each pass.
|
||||
if self.config.mla_scale_q_lora and not getattr(
|
||||
self_attn, "_mla_q_lora_scaled", False
|
||||
):
|
||||
self_attn.q_a_layernorm.weight.data *= (
|
||||
self.config.hidden_size / self.config.q_lora_rank
|
||||
) ** 0.5
|
||||
if self.config.mla_scale_kv_lora:
|
||||
self_attn._mla_q_lora_scaled = True
|
||||
if self.config.mla_scale_kv_lora and not getattr(
|
||||
self_attn, "_mla_kv_lora_scaled", False
|
||||
):
|
||||
self_attn.kv_a_layernorm.weight.data *= (
|
||||
self.config.hidden_size / self.config.kv_lora_rank
|
||||
) ** 0.5
|
||||
self_attn._mla_kv_lora_scaled = True
|
||||
return loaded_params
|
||||
|
||||
|
||||
|
||||
@@ -126,8 +126,10 @@ class LongCatMultiTokenPredictor(nn.Module):
|
||||
class LongCatFlashMTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
# LongCat MTP without MoE layers
|
||||
vllm_config.model_config.hf_config.n_routed_experts = None
|
||||
# LongCat MTP has no MoE layers: clear n_routed_experts so the predictor
|
||||
# builds a dense MLP. object.__setattr__ bypasses the ngram remote
|
||||
# config's strict validation (it rejects setting the int field to None).
|
||||
object.__setattr__(vllm_config.model_config.hf_config, "n_routed_experts", None)
|
||||
self.config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
|
||||
self.quant_config = (
|
||||
None
|
||||
@@ -287,14 +289,22 @@ class LongCatFlashMTP(nn.Module):
|
||||
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
|
||||
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
|
||||
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
|
||||
if self.config.mla_scale_q_lora:
|
||||
# Guard against compounding on incremental load_weights calls (the
|
||||
# in-place *= would otherwise double-apply the LoRA scaling).
|
||||
if self.config.mla_scale_q_lora and not getattr(
|
||||
self_attn, "_mla_q_lora_scaled", False
|
||||
):
|
||||
self_attn.q_a_layernorm.weight.data *= (
|
||||
self.config.hidden_size / self.config.q_lora_rank
|
||||
) ** 0.5
|
||||
if self.config.mla_scale_kv_lora:
|
||||
self_attn._mla_q_lora_scaled = True
|
||||
if self.config.mla_scale_kv_lora and not getattr(
|
||||
self_attn, "_mla_kv_lora_scaled", False
|
||||
):
|
||||
self_attn.kv_a_layernorm.weight.data *= (
|
||||
self.config.hidden_size / self.config.kv_lora_rank
|
||||
) ** 0.5
|
||||
self_attn._mla_kv_lora_scaled = True
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only LongCat-Flash-Lite (n-gram embedding) model.
|
||||
|
||||
``LongcatFlashNgramForCausalLM`` is LongCat-Flash (MLA dual-attention +
|
||||
zero-expert MoE + YaRN) plus an n-gram embedding input layer: each position's
|
||||
embedding fuses the token embedding with hashed embeddings of the preceding
|
||||
``n`` tokens. That per-request token history is isolated in a Model-Runner-V2
|
||||
:class:`LongcatNgramModelState` (mirroring ``DiffusionGemmaModelState``), so
|
||||
``get_model_state_cls`` makes the model MRV2-only.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .longcat_flash import FlashConfig, FlashModel
|
||||
from .utils import AutoWeightsLoader, PPMissingLayer, maybe_prefix
|
||||
|
||||
|
||||
def uses_ngram_embedding(config: FlashConfig) -> bool:
|
||||
return getattr(config, "ngram_vocab_size_ratio", None) is not None
|
||||
|
||||
|
||||
def _config_dtype(config: FlashConfig) -> torch.dtype:
|
||||
dt = getattr(config, "torch_dtype", None) or getattr(config, "dtype", None)
|
||||
if isinstance(dt, torch.dtype):
|
||||
return dt
|
||||
return getattr(torch, str(dt), None) or torch.bfloat16
|
||||
|
||||
|
||||
class NgramEmbedding(nn.Module):
|
||||
"""Token embedding fused with hashed n-gram embeddings.
|
||||
|
||||
TP-sharded: the ``k*(n-1)`` per-embedder tables are concatenated into one
|
||||
:class:`VocabParallelEmbedding` (``oe_embedder``) with per-embedder offsets,
|
||||
and the projections are stacked into one ``oe_projection`` applied with a
|
||||
single ``bmm``. Hashing math is ported from the HF reference.
|
||||
"""
|
||||
|
||||
def __init__(self, config: FlashConfig, base_embeddings: nn.Module) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.word_embeddings = base_embeddings
|
||||
|
||||
self.m = config.ngram_vocab_size_ratio * config.vocab_size
|
||||
self.k = config.emb_split_num
|
||||
self.n = config.emb_neighbor_num
|
||||
self.pad_id = config.pad_token_id
|
||||
self.eos_token_id = config.eos_token_id
|
||||
self._dtype = _config_dtype(config)
|
||||
|
||||
self._init_ngram_embeddings()
|
||||
|
||||
def _init_ngram_embeddings(self) -> None:
|
||||
self.num_embedders = self.k * (self.n - 1)
|
||||
oe_dim = self.config.hidden_size // self.num_embedders
|
||||
self.oe_dim = oe_dim
|
||||
|
||||
# Exclusive prefix sums of per-embedder table sizes; each embedder's
|
||||
# local id is offset into the single concatenated table.
|
||||
sizes = [int(self.m + i * 2 + 1) for i in range(self.num_embedders)]
|
||||
offsets = [0]
|
||||
for s in sizes:
|
||||
offsets.append(offsets[-1] + s)
|
||||
self._offsets = offsets # len num_embedders + 1
|
||||
self._sizes = sizes
|
||||
|
||||
self.oe_embedder = VocabParallelEmbedding(
|
||||
offsets[-1], oe_dim, params_dtype=self._dtype
|
||||
)
|
||||
# Stacked projections: oe_projection[i] = post_projs[i].weight.T
|
||||
self.oe_projection = nn.Parameter(
|
||||
torch.empty(
|
||||
self.num_embedders, oe_dim, self.config.hidden_size, dtype=self._dtype
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
# Precomputed tables for the CUDA n-gram id kernel (ngram_embedding
|
||||
# _kernels.cu): ne_weights[i][j][delta] = vocab^delta mod ne_mods[i][j],
|
||||
# ne_mods[i][j] = m + 2*(i*k+j) + 1. Registered as non-persistent buffers
|
||||
# so they follow the module to the device (not part of the checkpoint).
|
||||
vocab = self.config.vocab_size
|
||||
ne_weights = torch.zeros(self.n - 1, self.k, self.n, dtype=torch.int32)
|
||||
ne_mods = torch.zeros(self.n - 1, self.k, dtype=torch.int32)
|
||||
for i in range(self.n - 1):
|
||||
for j in range(self.k):
|
||||
mod = int(self.m + 2 * (i * self.k + j) + 1)
|
||||
ne_mods[i, j] = mod
|
||||
for delta in range(self.n):
|
||||
ne_weights[i, j, delta] = pow(vocab, delta, mod)
|
||||
self.register_buffer("ne_weights", ne_weights, persistent=False)
|
||||
self.register_buffer("ne_mods", ne_mods, persistent=False)
|
||||
self.register_buffer(
|
||||
"exclusive_sizes",
|
||||
torch.tensor(offsets, dtype=torch.int32),
|
||||
persistent=False,
|
||||
)
|
||||
|
||||
def load_weight(self, weight_name: str, loaded_weight: torch.Tensor) -> str:
|
||||
"""Split a per-embedder checkpoint weight into the sharded layout.
|
||||
|
||||
Returns the destination parameter's qualified name (relative to the
|
||||
enclosing model) so the caller can mark it loaded for completeness
|
||||
checks.
|
||||
"""
|
||||
if "ngram_embeddings.embedders." in weight_name:
|
||||
index = int(
|
||||
weight_name.split("ngram_embeddings.embedders.")[1].split(".")[0]
|
||||
)
|
||||
lo, hi = self._offsets[index], self._offsets[index + 1]
|
||||
assert hi - lo == loaded_weight.shape[0], (
|
||||
f"{hi - lo=} {loaded_weight.shape[0]=}"
|
||||
)
|
||||
shard = self.oe_embedder.shard_indices
|
||||
tp_start, tp_end = shard.org_vocab_start_index, shard.org_vocab_end_index
|
||||
load_start, load_end = max(lo, tp_start), min(hi, tp_end)
|
||||
if load_start < load_end:
|
||||
self.oe_embedder.weight.data[
|
||||
load_start - tp_start : load_end - tp_start
|
||||
] = loaded_weight[load_start - lo : load_end - lo]
|
||||
return "ngram_embeddings.oe_embedder.weight"
|
||||
elif "ngram_embeddings.post_projs." in weight_name:
|
||||
index = int(
|
||||
weight_name.split("ngram_embeddings.post_projs.")[1].split(".")[0]
|
||||
)
|
||||
self.oe_projection.data[index].copy_(loaded_weight.t())
|
||||
return "ngram_embeddings.oe_projection"
|
||||
else:
|
||||
raise AssertionError(f"Unexpected ngram weight: {weight_name}")
|
||||
|
||||
def embed_batched(
|
||||
self, input_ids: torch.Tensor, oe_ids: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Fused n-gram embedding for a flat batch given precomputed ids.
|
||||
|
||||
Args:
|
||||
input_ids: ``[num_tokens]`` current token per position.
|
||||
oe_ids: ``[num_tokens, num_embedders]`` global (offset) n-gram ids,
|
||||
as produced by the ``ngram_compute_n_gram_ids`` kernel.
|
||||
Returns: ``[num_tokens, hidden]``.
|
||||
"""
|
||||
word = self.word_embeddings(input_ids) # [N, H]
|
||||
flat = oe_ids.permute(1, 0).contiguous() # [num_embedders, N]
|
||||
oe = self.oe_embedder(flat) # [num_embedders, N, oe_dim]
|
||||
proj = torch.bmm(oe, self.oe_projection) # [num_embedders, N, H]
|
||||
all_h = torch.cat([word.unsqueeze(0), proj], dim=0) # [ne+1, N, H]
|
||||
return all_h.mean(dim=0) # [N, H]
|
||||
|
||||
|
||||
class FlashNgramModel(FlashModel):
|
||||
"""FlashModel whose input embedding is an :class:`NgramEmbedding`."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
# Each FlashDecoderLayer is a *dual* layer (2 attentions), so the number
|
||||
# of decoder layers is ``num_layers``. The ngram HF config sets
|
||||
# ``num_hidden_layers`` to a multiple of that (attention-module count),
|
||||
# which FlashModel would otherwise build as too many (dead) layers.
|
||||
hf = vllm_config.model_config.hf_config
|
||||
num_layers = getattr(hf, "num_layers", None)
|
||||
if num_layers is not None and hf.num_hidden_layers != num_layers:
|
||||
hf.num_hidden_layers = num_layers
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
if get_pp_group().is_first_rank and uses_ngram_embedding(self.config):
|
||||
self.ngram_embeddings = NgramEmbedding(self.config, self.embed_tokens)
|
||||
else:
|
||||
self.ngram_embeddings = None
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Names arrive with the ``model.`` prefix already stripped (routed here
|
||||
# by AutoWeightsLoader). Split the concatenated/sharded ngram tables and
|
||||
# stacked projections; delegate everything else to FlashModel.
|
||||
loaded: set[str] = set()
|
||||
rest: list[tuple[str, torch.Tensor]] = []
|
||||
for name, w in weights:
|
||||
if self.ngram_embeddings is not None and (
|
||||
"ngram_embeddings.embedders." in name
|
||||
or "ngram_embeddings.post_projs." in name
|
||||
):
|
||||
loaded.add(self.ngram_embeddings.load_weight(name, w))
|
||||
else:
|
||||
rest.append((name, w))
|
||||
loaded |= super().load_weights(rest)
|
||||
return loaded
|
||||
|
||||
|
||||
class LongcatFlashNgramForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
"""LongCat-Flash-Lite for causal LM (MRV2-only, n-gram embedding)."""
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
if not vllm_config.use_v2_model_runner:
|
||||
raise NotImplementedError(
|
||||
"LongcatFlashNgramForCausalLM (LongCat-Flash-Lite) requires the "
|
||||
"V2 model runner for its n-gram embedding state; it is selected "
|
||||
"automatically unless VLLM_USE_V2_MODEL_RUNNER=0 is set."
|
||||
)
|
||||
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
|
||||
config.intermediate_size = getattr(
|
||||
config, "ffn_hidden_size", config.intermediate_size
|
||||
)
|
||||
self.config = config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
self.model = FlashNgramModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_model_state_cls() -> type["LongcatNgramModelState"]:
|
||||
return LongcatNgramModelState
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
):
|
||||
# inputs_embeds is produced by LongcatNgramModelState.prepare_inputs.
|
||||
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def get_expert_mapping(self):
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# AutoWeightsLoader routes ``model.*`` to FlashNgramModel.load_weights
|
||||
# (which handles the ngram split) and ``lm_head.*`` to the head. MTP
|
||||
# weights are not part of this model.
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=["model.mtp."])
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
class LongcatNgramModelState(DefaultModelState):
|
||||
"""Per-request n-gram token history for LongCat-Flash-Lite.
|
||||
|
||||
Maintains a small CPU-side per-slot context (last ``n-1`` processed tokens)
|
||||
and a persistent ``inputs_embeds`` buffer. ``prepare_inputs`` computes the
|
||||
fused n-gram embedding per request into the buffer, handed to the model
|
||||
forward as ``inputs_embeds``.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config, model, encoder_cache, device) -> None:
|
||||
super().__init__(vllm_config, model, encoder_cache, device)
|
||||
config = model.config
|
||||
self.ngram = model.model.ngram_embeddings
|
||||
self.n = int(config.emb_neighbor_num)
|
||||
self.ctx_len = self.n - 1
|
||||
self.eos_id = int(config.eos_token_id)
|
||||
|
||||
# Per-slot left-context: last ``n-1`` processed tokens, EOS negated. A
|
||||
# negative entry (incl. the -1 fill) marks a context boundary that stops
|
||||
# the n-gram walk (matches the kernel's EOS break / fresh-request start).
|
||||
self.token_context = torch.full(
|
||||
(self.max_num_reqs, self.ctx_len), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
self._inputs_embeds_buf = torch.zeros(
|
||||
self.max_num_tokens,
|
||||
config.hidden_size,
|
||||
dtype=self.dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def _neg_eos(self, toks: list[int]) -> list[int]:
|
||||
return [-t if t == self.eos_id else t for t in toks]
|
||||
|
||||
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
|
||||
super().add_request(req_index, new_req_data) # rope positions
|
||||
# Fresh request -> no left-context (-1 fill). On resume, seed from the
|
||||
# already-processed token tail. Use prefill_token_ids (full processed
|
||||
# sequence incl. generated tokens on v2 resume), like DefaultModelState;
|
||||
# the prompt alone would be too short when resuming after decode.
|
||||
ctx = [-1] * self.ctx_len
|
||||
ncomp = new_req_data.num_computed_tokens
|
||||
toks_src = new_req_data.prefill_token_ids or new_req_data.prompt_token_ids
|
||||
if ncomp > 0 and toks_src is not None:
|
||||
lo = max(0, ncomp - self.ctx_len)
|
||||
toks = self._neg_eos(list(toks_src[lo:ncomp]))
|
||||
ctx[self.ctx_len - len(toks) :] = toks
|
||||
self.token_context[req_index] = torch.tensor(
|
||||
ctx, dtype=torch.int32, device=self.token_context.device
|
||||
)
|
||||
|
||||
def prepare_inputs(
|
||||
self, input_batch: InputBatch, req_states: RequestState
|
||||
) -> dict[str, Any]:
|
||||
model_inputs = super().prepare_inputs(input_batch, req_states) # positions
|
||||
num_tokens = input_batch.num_tokens
|
||||
num_padded = input_batch.num_tokens_after_padding
|
||||
input_ids = input_batch.input_ids[:num_tokens]
|
||||
embeds = self._inputs_embeds_buf[:num_padded]
|
||||
|
||||
oe_ids = self._compute_oe_ids(input_batch)
|
||||
embeds[:num_tokens].copy_(self.ngram.embed_batched(input_ids, oe_ids))
|
||||
model_inputs["inputs_embeds"] = embeds
|
||||
return model_inputs
|
||||
|
||||
def prepare_dummy_inputs(self, num_reqs: int, num_tokens: int) -> dict[str, Any]:
|
||||
# FULL cudagraph replay reads only the captured buffers, so capture must
|
||||
# reference the same persistent ``inputs_embeds`` buffer prepare_inputs
|
||||
# re-fills (the base class wires this for multimodal models only).
|
||||
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens) # positions
|
||||
model_inputs["inputs_embeds"] = self._inputs_embeds_buf[:num_tokens]
|
||||
return model_inputs
|
||||
|
||||
def _compute_oe_ids(self, input_batch: InputBatch) -> torch.Tensor:
|
||||
"""Batched global n-gram ids ``[num_tokens, num_embedders]``.
|
||||
|
||||
Assembles an ephemeral per-request token table (``[n-1] context ++
|
||||
current tokens``, EOS-negated) and runs the ``ngram_compute_n_gram_ids``
|
||||
CUDA kernel for the whole batch, then rolls each slot's context forward.
|
||||
"""
|
||||
device = self.token_context.device
|
||||
num_tokens = input_batch.num_tokens
|
||||
num_reqs = input_batch.num_reqs
|
||||
ctx_len = self.ctx_len
|
||||
idx_mapping = input_batch.idx_mapping[:num_reqs].long()
|
||||
qsl = input_batch.query_start_loc[: num_reqs + 1].to(torch.int32)
|
||||
cur = input_batch.input_ids[:num_tokens].to(torch.int32)
|
||||
|
||||
cur_neg = torch.where(cur == self.eos_id, -cur, cur)
|
||||
req_lens = qsl[1:] - qsl[:-1]
|
||||
max_len = int(req_lens.max().item())
|
||||
width = ctx_len + max_len
|
||||
|
||||
# table[r] = [context(n-1) | current tokens | pad(-1)]
|
||||
table = torch.full((num_reqs, width), -1, dtype=torch.int32, device=device)
|
||||
table[:, :ctx_len] = self.token_context[idx_mapping]
|
||||
tok_req = torch.repeat_interleave(
|
||||
torch.arange(num_reqs, device=device), req_lens.long()
|
||||
)
|
||||
col = ctx_len + (
|
||||
torch.arange(num_tokens, device=device) - qsl[:-1].long()[tok_req]
|
||||
)
|
||||
table[tok_req, col] = cur_neg
|
||||
|
||||
column_starts = torch.full(
|
||||
(num_reqs,), ctx_len, dtype=torch.int32, device=device
|
||||
)
|
||||
row_indices = torch.arange(num_reqs, dtype=torch.int64, device=device)
|
||||
n_gram_ids = torch.empty(
|
||||
num_tokens, self.ngram.num_embedders, dtype=torch.int32, device=device
|
||||
)
|
||||
ops.ngram_compute_n_gram_ids(
|
||||
self.n,
|
||||
self.ngram.k,
|
||||
self.ngram.ne_weights,
|
||||
self.ngram.ne_mods,
|
||||
self.ngram.exclusive_sizes,
|
||||
qsl,
|
||||
table,
|
||||
row_indices,
|
||||
column_starts,
|
||||
n_gram_ids,
|
||||
)
|
||||
|
||||
# Roll context: new context = last n-1 of [context | current] per slot.
|
||||
gather = req_lens.long().unsqueeze(1) + torch.arange(
|
||||
ctx_len, device=device
|
||||
).unsqueeze(0)
|
||||
rows = torch.arange(num_reqs, device=device).unsqueeze(1)
|
||||
self.token_context[idx_mapping] = table[rows, gather]
|
||||
return n_gram_ids.long()
|
||||
@@ -145,6 +145,10 @@ _TEXT_GENERATION_MODELS = {
|
||||
# For decapoda-research/llama-*
|
||||
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
|
||||
"LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
|
||||
"LongcatFlashNgramForCausalLM": (
|
||||
"longcat_flash_ngram",
|
||||
"LongcatFlashNgramForCausalLM",
|
||||
),
|
||||
"MambaForCausalLM": ("mamba", "MambaForCausalLM"),
|
||||
"Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
|
||||
"MellumForCausalLM": ("mellum", "MellumForCausalLM"),
|
||||
|
||||
@@ -268,6 +268,7 @@ class ModelArchConfigConvertorBase:
|
||||
"kimi_k2",
|
||||
"kimi_linear",
|
||||
"longcat_flash",
|
||||
"longcat_flash_ngram",
|
||||
"pangu_ultra_moe",
|
||||
"pangu_ultra_moe_mtp",
|
||||
"bailing_hybrid",
|
||||
|
||||
@@ -540,7 +540,15 @@ def init_kv_cache(
|
||||
shared_kv_cache_layers=shared_kv_cache_layers,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
bind_kv_cache(kv_caches, forward_context, runner_kv_caches)
|
||||
# Dual-attention models (e.g. LongCat-Flash) put two Attention modules per
|
||||
# decoder layer, so a layer name carries two integers (layer + module index).
|
||||
num_attn_module = (
|
||||
2
|
||||
if vllm_config.model_config.hf_config.model_type
|
||||
in ("longcat_flash", "longcat_flash_ngram")
|
||||
else 1
|
||||
)
|
||||
bind_kv_cache(kv_caches, forward_context, runner_kv_caches, num_attn_module)
|
||||
return kv_caches
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user