[Perf][DSpark] Add KV-only context insertion across V4.1 cache formats (#56441)
Signed-off-by: 0z5a <0z5a@users.noreply.github.com> Signed-off-by: Yongye Zhu <yongye@inferact.ai> Co-authored-by: 0z5a <0z5a@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
0z5a
Codex
Yongye Zhu
Claude Opus 5
parent
a31ec3a68b
commit
6ecd97f1b7
@@ -657,7 +657,12 @@ static void launchFusedDeepseekV4Templated(
|
||||
config.attrs = attrs;
|
||||
config.numAttrs = (sm_version >= 90) ? 1 : 0;
|
||||
|
||||
if (num_tokens_full < NUM_TOKEN_CUTOFF) {
|
||||
// The reduced grid pays off by iterating a token's Q heads inside one CTA.
|
||||
// With zero query heads there is nothing to iterate, so one warp per block
|
||||
// would do all the work and the other seven idle; keep the warp-per-slot
|
||||
// grid instead. Only reachable above NUM_TOKEN_CUTOFF tokens in a single
|
||||
// insert, where it is worth ~2x.
|
||||
if (kNumHeadsQPadded == 0 || num_tokens_full < NUM_TOKEN_CUTOFF) {
|
||||
cudaLaunchKernelEx(&config,
|
||||
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
|
||||
@@ -993,6 +998,86 @@ static void launchFullCacheKernel(
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Torch op wrapper
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
void fused_deepseek_v4_kv_rope_insert(
|
||||
torch::stable::Tensor const& kv, torch::stable::Tensor& k_cache,
|
||||
torch::stable::Tensor const& slot_mapping,
|
||||
torch::stable::Tensor const& position_ids,
|
||||
torch::stable::Tensor const& cos_sin_cache, int64_t cache_block_size,
|
||||
std::optional<torch::stable::Tensor> fp8_scale, bool kv_mxfp8) {
|
||||
using torch::headeronly::ScalarType;
|
||||
STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous() &&
|
||||
kv.dim() == 2 && kv.size(1) == 512 &&
|
||||
kv.scalar_type() == ScalarType::BFloat16,
|
||||
"kv must be contiguous CUDA bfloat16 [tokens, 512]");
|
||||
auto const device = kv.get_device_index();
|
||||
STD_TORCH_CHECK(k_cache.device().is_cuda() &&
|
||||
k_cache.get_device_index() == device &&
|
||||
k_cache.dim() == 3 && cache_block_size > 0 &&
|
||||
k_cache.size(1) == cache_block_size &&
|
||||
k_cache.stride(2) == 1,
|
||||
"k_cache must be a paged cache on the KV device");
|
||||
for (auto const* tensor : {&slot_mapping, &position_ids}) {
|
||||
STD_TORCH_CHECK(tensor->device().is_cuda() &&
|
||||
tensor->get_device_index() == device &&
|
||||
tensor->scalar_type() == ScalarType::Long &&
|
||||
tensor->dim() == 1 && tensor->is_contiguous(),
|
||||
"slots and positions must be contiguous int64 on the KV device");
|
||||
}
|
||||
STD_TORCH_CHECK(position_ids.size(0) == kv.size(0) &&
|
||||
slot_mapping.size(0) <= kv.size(0),
|
||||
"positions must match KV rows; slots may omit padded rows");
|
||||
STD_TORCH_CHECK(cos_sin_cache.device().is_cuda() &&
|
||||
cos_sin_cache.get_device_index() == device &&
|
||||
cos_sin_cache.scalar_type() == ScalarType::Float &&
|
||||
cos_sin_cache.dim() == 2 &&
|
||||
cos_sin_cache.size(1) == 64 && cos_sin_cache.is_contiguous(),
|
||||
"cos_sin_cache must be contiguous float32 [max_pos, 64]");
|
||||
bool const packed = k_cache.scalar_type() == ScalarType::Byte;
|
||||
bool const fp8 = k_cache.scalar_type() == ScalarType::Float8_e4m3fn;
|
||||
STD_TORCH_CHECK(packed || fp8 ||
|
||||
k_cache.scalar_type() == ScalarType::BFloat16,
|
||||
"cache must be uint8, bfloat16 or float8_e4m3fn");
|
||||
STD_TORCH_CHECK(k_cache.size(2) == (packed ? (kv_mxfp8 ? 528 : 584) : 512),
|
||||
"cache row width does not match its dtype");
|
||||
if (fp8) {
|
||||
STD_TORCH_CHECK(fp8_scale.has_value() && fp8_scale->device().is_cuda() &&
|
||||
fp8_scale->get_device_index() == device &&
|
||||
fp8_scale->scalar_type() == ScalarType::Float &&
|
||||
fp8_scale->numel() == 1,
|
||||
"fp8 cache requires one float32 KV scale on the KV device");
|
||||
}
|
||||
int const num_tokens = static_cast<int>(slot_mapping.size(0));
|
||||
if (num_tokens == 0) return;
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(device);
|
||||
const cudaStream_t stream = get_current_cuda_stream(device);
|
||||
using scalar_t = c10::BFloat16;
|
||||
auto const* input = reinterpret_cast<scalar_t const*>(kv.const_data_ptr());
|
||||
auto* cache = reinterpret_cast<uint8_t*>(k_cache.mutable_data_ptr());
|
||||
auto const* slots = slot_mapping.const_data_ptr<int64_t>();
|
||||
auto const* positions = position_ids.const_data_ptr<int64_t>();
|
||||
auto const* cos_sin = cos_sin_cache.const_data_ptr<float>();
|
||||
// Zero query heads schedules only the existing KV branch, preserving its
|
||||
// RoPE rounding and quantization without allocating any query tensors.
|
||||
if (packed) {
|
||||
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0, false>(
|
||||
nullptr, nullptr, input, cache, slots, positions, cos_sin, 0.0f,
|
||||
num_tokens, num_tokens, 0, static_cast<int>(cache_block_size),
|
||||
static_cast<int>(k_cache.stride(0)), kv_mxfp8, stream);
|
||||
} else if (fp8) {
|
||||
vllm::deepseek_v4_fused_ops::launchFullCacheKernel<scalar_t, false, true>(
|
||||
nullptr, nullptr, 0, 0, input, cache, slots, positions, cos_sin,
|
||||
fp8_scale->const_data_ptr<float>(), nullptr, 0.0f, num_tokens, num_tokens,
|
||||
0, static_cast<int>(cache_block_size), k_cache.stride(0),
|
||||
k_cache.stride(1), false, "fused_deepseek_v4_kv_rope_insert", stream);
|
||||
} else {
|
||||
vllm::deepseek_v4_fused_ops::launchFullCacheKernel<scalar_t, false, false>(
|
||||
nullptr, nullptr, 0, 0, input, cache, slots, positions, cos_sin, nullptr,
|
||||
nullptr, 0.0f, num_tokens, num_tokens, 0, static_cast<int>(cache_block_size),
|
||||
k_cache.stride(0) * 2, k_cache.stride(1) * 2, false,
|
||||
"fused_deepseek_v4_kv_rope_insert", stream);
|
||||
}
|
||||
}
|
||||
|
||||
torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
torch::stable::Tensor const& q_in, // [N, num_heads_q, 512] bf16
|
||||
torch::stable::Tensor const& kv, // [N, 512] bf16 (read-only)
|
||||
|
||||
@@ -259,6 +259,13 @@ void fused_qk_norm_rope(torch::stable::Tensor& qkv, int64_t num_heads_q,
|
||||
torch::stable::Tensor& position_ids,
|
||||
int64_t forced_token_heads_per_warp);
|
||||
|
||||
void fused_deepseek_v4_kv_rope_insert(
|
||||
torch::stable::Tensor const& kv, torch::stable::Tensor& k_cache,
|
||||
torch::stable::Tensor const& slot_mapping,
|
||||
torch::stable::Tensor const& position_ids,
|
||||
torch::stable::Tensor const& cos_sin_cache, int64_t cache_block_size,
|
||||
std::optional<torch::stable::Tensor> fp8_scale, bool kv_mxfp8);
|
||||
|
||||
torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
torch::stable::Tensor const& q_in, torch::stable::Tensor const& kv,
|
||||
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
|
||||
|
||||
@@ -425,6 +425,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"bool is_neox, Tensor position_ids, "
|
||||
"int forced_token_heads_per_warp=-1) -> ()");
|
||||
|
||||
ops.def(
|
||||
"fused_deepseek_v4_kv_rope_insert("
|
||||
"Tensor kv, Tensor! k_cache, Tensor slot_mapping, Tensor position_ids, "
|
||||
"Tensor cos_sin_cache, int cache_block_size, Tensor? fp8_scale=None, "
|
||||
"bool kv_mxfp8=False) -> "
|
||||
"()");
|
||||
ops.def(
|
||||
"fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert("
|
||||
"Tensor q_in, Tensor kv, Tensor! k_cache, "
|
||||
@@ -807,6 +813,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
// Positional encoding kernels (shared CUDA/ROCm)
|
||||
ops.impl("rotary_embedding", TORCH_BOX(&rotary_embedding));
|
||||
ops.impl("fused_qk_norm_rope", TORCH_BOX(&fused_qk_norm_rope));
|
||||
ops.impl("fused_deepseek_v4_kv_rope_insert",
|
||||
TORCH_BOX(&fused_deepseek_v4_kv_rope_insert));
|
||||
ops.impl("fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert",
|
||||
TORCH_BOX(&fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert));
|
||||
ops.impl(
|
||||
|
||||
Reference in New Issue
Block a user