This commit is contained in:
GeeeekExplorer
2025-06-10 08:52:58 +08:00
parent a5a4909e6a
commit b98e1ca305
10 changed files with 39 additions and 26 deletions

View File

@@ -64,10 +64,7 @@ class Sequence:
def last_block(self, block_size=256):
n = self.num_blocks
t = len(self) + block_size - self.num_blocks * block_size
x = self.token_ids[(n-1)*block_size:]
assert len(x) == t
return x
return self.token_ids[(n-1)*block_size:]
def append_token(self, token_id: int):
self.token_ids.append(token_id)