[V1][Kernel] Flashinfer HND KV cache layout (#19280)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -128,6 +128,7 @@ if TYPE_CHECKING:
|
||||
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
|
||||
VLLM_SLEEP_WHEN_IDLE: bool = False
|
||||
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
|
||||
VLLM_KV_CACHE_LAYOUT: Optional[str] = None
|
||||
|
||||
|
||||
def get_default_cache_root():
|
||||
@@ -879,6 +880,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# processes via zmq.
|
||||
"VLLM_MQ_MAX_CHUNK_BYTES_MB":
|
||||
lambda: int(os.getenv("VLLM_MQ_MAX_CHUNK_BYTES_MB", "16")),
|
||||
|
||||
# KV Cache layout used throughout vllm.
|
||||
# Some common values are:
|
||||
# - NHD
|
||||
# - HND
|
||||
# Where N=num_blocks, H=num_heads and D=head_size. The default value will
|
||||
# leave the layout choice to the backend. Mind that backends may only
|
||||
# implement and support a subset of all possible layouts.
|
||||
"VLLM_KV_CACHE_LAYOUT":
|
||||
lambda: os.getenv("VLLM_KV_CACHE_LAYOUT", None)
|
||||
}
|
||||
|
||||
# --8<-- [end:env-vars-definition]
|
||||
|
||||
Reference in New Issue
Block a user