<|endoftext|> token isn't encoded correctly
Author: ttumielCreated Jun 27, 2024Updated Sep 28, 2024
import torch
from mingpt.bpe import BPETokenizer
tokenizer = BPETokenizer()
print(tokenizer("<|endoftext|>")) # tensor([[ 27, 91, 437, 1659, 5239, 91, 29]])
print(tokenizer.decode(torch.tensor([50256]))) # '<|endoftext|>'
print(tokenizer(tokenizer.decode(torch.tensor([50256])))) # tensor([[ 27, 91, 437, 1659, 5239, 91, 29]])Source: karpathy/minGPT