How to add byte_fallback tokens?

from tokenizers import Tokenizer
from tokenizers import decoders, pre_tokenizers
from tokenizers.models import BPE
from tokenizers.processors import TemplateProcessing
from tokenizers.trainers import BpeTrainer
from tokenizers import AddedToken

from datasets import load_dataset

dataset = load_dataset("tapaco")

def topaco_generator():
    for i in dataset['train']:
        yield i['paraphrase']

bpe_trainer = BpeTrainer(
    special_tokens=["<unk>", "<s>", "</s>", "<pad>"]
    + [f"<0x{i:02X}>" for i in range(256)]  # byte_fallback tokens
)

tokenizer = Tokenizer(BPE(byte_fallback=True))
tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
    [pre_tokenizers.Metaspace(), pre_tokenizers.Digits(individual_digits=True)]
)
tokenizer.enable_padding(pad_id=3, pad_token="<pad>")
tokenizer.post_processor = TemplateProcessing(
    single="<s> $A </s>",
    pair="<s> $A </s> $B </s>",
    special_tokens=[
        ("<s>", 1),
        ("</s>", 2),
    ],
)
tokenizer.decoder = decoders.Sequence(
    [
        decoders.Metaspace(),
        decoders.ByteFallback(),
    ]
)
# my attempt to add byte_fallback as non-special tokens
# tokenizer.add_tokens([AddedToken(content=f"<0x{i:02X}>", special=True, normalized=False) for i in range(256)])

tokenizer.train_from_iterator(topaco_generator(), trainer=bpe_trainer)
tokenizer.save("topaco_tokenizer.json")

tokenizer = Tokenizer.from_file("topaco_tokenizer.json")

text = "I love you more than I can say "
encoded_text = tokenizer.encode(text)
print(encoded_text.tokens)
# My work around to preverse byte_fallback tokens
# and remove other special tokens
decoded_text = tokenizer.decode(encoded_text.ids, skip_special_tokens=False)
print(decoded_text.removeprefix('<s> ').removesuffix('</s>'))