[HMA]Fix corner case when hybrid page_size can not be evenly divided issue (blk_size=64,tp=4) (#37467)
Signed-off-by: Chendi Xue <chendi.xue@intel.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Chendi.Xue <chendi.xue@intel.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Nicolò Lucchesi
parent
b4a2f3ac36
commit
3b1dbaad4e
@@ -38,6 +38,8 @@ class CacheConfig:
|
||||
Accepts None (meaning "use default"). After construction, always int."""
|
||||
user_specified_block_size: bool = field(default=False, init=False)
|
||||
"""Whether block_size was explicitly provided. Derived automatically."""
|
||||
user_specified_mamba_block_size: bool = field(default=False, init=False)
|
||||
"""Whether mamba_block_size was explicitly provided. Derived automatically."""
|
||||
gpu_memory_utilization: float = Field(default=0.9, gt=0, le=1)
|
||||
"""The fraction of GPU memory to be used for the model executor, which can
|
||||
range from 0 to 1. For example, a value of 0.5 would imply 50% GPU memory
|
||||
@@ -182,6 +184,7 @@ class CacheConfig:
|
||||
"cpu_kvcache_space_bytes",
|
||||
"mamba_page_size_padded",
|
||||
"user_specified_block_size",
|
||||
"user_specified_mamba_block_size",
|
||||
"_block_size_resolved",
|
||||
# Post-init/derived counters
|
||||
"num_gpu_blocks",
|
||||
@@ -214,6 +217,8 @@ class CacheConfig:
|
||||
object.__setattr__(self, "block_size", self.DEFAULT_BLOCK_SIZE)
|
||||
else:
|
||||
object.__setattr__(self, "user_specified_block_size", True)
|
||||
if self.mamba_block_size is not None:
|
||||
object.__setattr__(self, "user_specified_mamba_block_size", True)
|
||||
return self
|
||||
|
||||
@field_validator("calculate_kv_scales", mode="after")
|
||||
|
||||
Reference in New Issue
Block a user