Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 17:41:13 +00:00
1122 changed files with 60114 additions and 20956 deletions
+12
View File
@@ -50,6 +50,7 @@ CacheDType = Literal[
"turboquant_4bit_nc",
"turboquant_k3v4_nc",
"turboquant_3bit_nc",
"ultraquant_4bit",
"int4_per_token_head",
"int8_per_token_head",
"fp8_per_token_head",
@@ -229,6 +230,15 @@ class CacheConfig:
for hybrid models where requests occupy multiple KV cache groups."""
kv_cache_max_concurrency: float | None = field(default=None, init=False)
"""Per-DP-engine maximum concurrency at max_model_len tokens."""
# The two differ when hashes are finer than the scheduler block but partial
# hash hits are off (e.g. a sliding-window group cannot look up sub-block
# hits), so hits only land on the scheduler block.
# TODO(zjy0516): Merge into one value
hash_block_size: int | None = field(default=None, init=False)
"""Tokens per prefix-cache block hash."""
cache_hit_alignment_tokens: int | None = field(default=None, init=False)
"""Token granularity at which prefix-cache hits land: `hash_block_size`
with partial hash hits, otherwise the scheduler block size."""
kv_sharing_fast_prefill: bool = False
"""In some KV sharing setups, e.g. YOCO (https://arxiv.org/abs/2405.05254),
@@ -297,6 +307,8 @@ class CacheConfig:
"effective_attention_block_size",
"kv_cache_size_tokens",
"kv_cache_max_concurrency",
"hash_block_size",
"cache_hit_alignment_tokens",
# Feature toggles not impacting compiled graph shape
"kv_sharing_fast_prefill",
"swa_bounded_replay",