[feat] Need to optimized with async prefetch.

2025-12-15 06:58:40 +08:00
parent 1081ab51ea
commit b8b6478506
9 changed files with 556 additions and 404 deletions
--- a/nanovllm/config.py
+++ b/nanovllm/config.py
@@ -14,7 +14,7 @@ class Config:
    enforce_eager: bool = False
    hf_config: AutoConfig | None = None
    eos: int = -1
-    kvcache_block_size: int = 256
+    kvcache_block_size: int = 4096
    num_kvcache_blocks: int = -1

    # CPU Offload configuration