在 datasets.map 之后, AutoTokenizer 的哈希值发生了变化

from transformers import AutoTokenizer, BertTokenizer from datasets import load_dataset from datasets.fingerprint import Hasher tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

def tokenize_function(example): return tokenizer(example["sentence1"], example["sentence2"], truncation=True)

raw_datasets = load_dataset("glue", "mrpc")

print(Hasher.hash(tokenize_function)) print(Hasher.hash(tokenizer))

tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)

print(Hasher.hash(tokenize_function)) print(Hasher.hash(tokenizer))

Reusing dataset glue (/home1/wts/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:00<00:00, 1091.22it/s]
46d4b31f54153fc7
5b8771afd8d43888
Loading cached