[feat] Need to optimized with async prefetch.

2025-12-15 06:58:40 +08:00
parent 1081ab51ea
commit b8b6478506
9 changed files with 556 additions and 404 deletions
--- a/nanovllm/engine/sequence.py
+++ b/nanovllm/engine/sequence.py
@@ -12,7 +12,7 @@ class SequenceStatus(Enum):


 class Sequence:
-    block_size = 256
+    block_size = 4096
    counter = count()

    def __init__(self, token_ids: list[int], sampling_params = SamplingParams()):